blob: 6f51d9c1879d86c4a4e57da8185e8e527100abd7 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001SSTATE_VERSION = "3"
2
3SSTATE_MANIFESTS ?= "${TMPDIR}/sstate-control"
4SSTATE_MANFILEPREFIX = "${SSTATE_MANIFESTS}/manifest-${SSTATE_MANMACH}-${PN}"
5
6def generate_sstatefn(spec, hash, d):
7 if not hash:
8 hash = "INVALID"
9 return hash[:2] + "/" + spec + hash
10
11SSTATE_PKGARCH = "${PACKAGE_ARCH}"
12SSTATE_PKGSPEC = "sstate:${PN}:${PACKAGE_ARCH}${TARGET_VENDOR}-${TARGET_OS}:${PV}:${PR}:${SSTATE_PKGARCH}:${SSTATE_VERSION}:"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050013SSTATE_SWSPEC = "sstate:${PN}::${PV}:${PR}::${SSTATE_VERSION}:"
Brad Bishop19323692019-04-05 15:28:33 -040014SSTATE_PKGNAME = "${SSTATE_EXTRAPATH}${@generate_sstatefn(d.getVar('SSTATE_PKGSPEC'), d.getVar('BB_UNIHASH'), d)}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050015SSTATE_PKG = "${SSTATE_DIR}/${SSTATE_PKGNAME}"
16SSTATE_EXTRAPATH = ""
17SSTATE_EXTRAPATHWILDCARD = ""
18SSTATE_PATHSPEC = "${SSTATE_DIR}/${SSTATE_EXTRAPATHWILDCARD}*/${SSTATE_PKGSPEC}"
19
Patrick Williamsc0f7c042017-02-23 20:41:17 -060020# explicitly make PV to depend on evaluated value of PV variable
21PV[vardepvalue] = "${PV}"
22
Patrick Williamsc124f4f2015-09-15 14:41:29 -050023# We don't want the sstate to depend on things like the distro string
24# of the system, we let the sstate paths take care of this.
25SSTATE_EXTRAPATH[vardepvalue] = ""
Brad Bishop19323692019-04-05 15:28:33 -040026SSTATE_EXTRAPATHWILDCARD[vardepvalue] = ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -050027
28# For multilib rpm the allarch packagegroup files can overwrite (in theory they're identical)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080029SSTATE_DUPWHITELIST = "${DEPLOY_DIR}/licenses/"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050030# Avoid docbook/sgml catalog warnings for now
31SSTATE_DUPWHITELIST += "${STAGING_ETCDIR_NATIVE}/sgml ${STAGING_DATADIR_NATIVE}/sgml"
Brad Bishop316dfdd2018-06-25 12:45:53 -040032# sdk-provides-dummy-nativesdk and nativesdk-buildtools-perl-dummy overlap for different SDKMACHINE
33SSTATE_DUPWHITELIST += "${DEPLOY_DIR_RPM}/sdk_provides_dummy_nativesdk/ ${DEPLOY_DIR_IPK}/sdk-provides-dummy-nativesdk/"
34SSTATE_DUPWHITELIST += "${DEPLOY_DIR_RPM}/buildtools_dummy_nativesdk/ ${DEPLOY_DIR_IPK}/buildtools-dummy-nativesdk/"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080035# target-sdk-provides-dummy overlaps that allarch is disabled when multilib is used
36SSTATE_DUPWHITELIST += "${COMPONENTS_DIR}/sdk-provides-dummy-target/ ${DEPLOY_DIR_RPM}/sdk_provides_dummy_target/ ${DEPLOY_DIR_IPK}/sdk-provides-dummy-target/"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050037# Archive the sources for many architectures in one deploy folder
38SSTATE_DUPWHITELIST += "${DEPLOY_DIR_SRC}"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080039# ovmf/grub-efi/systemd-boot/intel-microcode multilib recipes can generate identical overlapping files
40SSTATE_DUPWHITELIST += "${DEPLOY_DIR_IMAGE}/ovmf"
41SSTATE_DUPWHITELIST += "${DEPLOY_DIR_IMAGE}/grub-efi"
42SSTATE_DUPWHITELIST += "${DEPLOY_DIR_IMAGE}/systemd-boot"
43SSTATE_DUPWHITELIST += "${DEPLOY_DIR_IMAGE}/microcode"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050044
Brad Bishop6e60e8b2018-02-01 10:27:11 -050045SSTATE_SCAN_FILES ?= "*.la *-config *_config postinst-*"
46SSTATE_SCAN_CMD ??= 'find ${SSTATE_BUILDDIR} \( -name "${@"\" -o -name \"".join(d.getVar("SSTATE_SCAN_FILES").split())}" \) -type f'
47SSTATE_SCAN_CMD_NATIVE ??= 'grep -Irl -e ${RECIPE_SYSROOT} -e ${RECIPE_SYSROOT_NATIVE} -e ${HOSTTOOLS_DIR} ${SSTATE_BUILDDIR}'
Patrick Williamsc124f4f2015-09-15 14:41:29 -050048
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050049BB_HASHFILENAME = "False ${SSTATE_PKGSPEC} ${SSTATE_SWSPEC}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050050
51SSTATE_ARCHS = " \
52 ${BUILD_ARCH} \
53 ${BUILD_ARCH}_${SDK_ARCH}_${SDK_OS} \
54 ${BUILD_ARCH}_${TARGET_ARCH} \
55 ${SDK_ARCH}_${SDK_OS} \
56 ${SDK_ARCH}_${PACKAGE_ARCH} \
57 allarch \
58 ${PACKAGE_ARCH} \
Brad Bishop316dfdd2018-06-25 12:45:53 -040059 ${PACKAGE_EXTRA_ARCHS} \
60 ${MACHINE_ARCH}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050061
62SSTATE_MANMACH ?= "${SSTATE_PKGARCH}"
63
64SSTATECREATEFUNCS = "sstate_hardcode_path"
Brad Bishop19323692019-04-05 15:28:33 -040065SSTATECREATEFUNCS[vardeps] = "SSTATE_SCAN_FILES"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050066SSTATEPOSTCREATEFUNCS = ""
67SSTATEPREINSTFUNCS = ""
68SSTATEPOSTUNPACKFUNCS = "sstate_hardcode_path_unpack"
69SSTATEPOSTINSTFUNCS = ""
Brad Bishop6e60e8b2018-02-01 10:27:11 -050070EXTRA_STAGING_FIXMES ?= "HOSTTOOLS_DIR"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050071
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050072# Check whether sstate exists for tasks that support sstate and are in the
73# locked signatures file.
74SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK ?= 'error'
75
76# Check whether the task's computed hash matches the task's hash in the
77# locked signatures file.
78SIGGEN_LOCKEDSIGS_TASKSIG_CHECK ?= "error"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050079
80# The GnuPG key ID and passphrase to use to sign sstate archives (or unset to
81# not sign)
82SSTATE_SIG_KEY ?= ""
83SSTATE_SIG_PASSPHRASE ?= ""
84# Whether to verify the GnUPG signatures when extracting sstate archives
85SSTATE_VERIFY_SIG ?= "0"
86
Brad Bishop19323692019-04-05 15:28:33 -040087SSTATE_HASHEQUIV_METHOD ?= "oe.sstatesig.OEOuthashBasic"
88SSTATE_HASHEQUIV_METHOD[doc] = "The fully-qualified function used to calculate \
89 the output hash for a task, which in turn is used to determine equivalency. \
90 "
91
92SSTATE_HASHEQUIV_SERVER ?= ""
93SSTATE_HASHEQUIV_SERVER[doc] = "The hash equivalence sever. For example, \
94 'http://192.168.0.1:5000'. Do not include a trailing slash \
95 "
96
97SSTATE_HASHEQUIV_REPORT_TASKDATA ?= "0"
98SSTATE_HASHEQUIV_REPORT_TASKDATA[doc] = "Report additional useful data to the \
99 hash equivalency server, such as PN, PV, taskname, etc. This information \
100 is very useful for developers looking at task data, but may leak sensitive \
101 data if the equivalence server is public. \
102 "
103
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500104python () {
105 if bb.data.inherits_class('native', d):
106 d.setVar('SSTATE_PKGARCH', d.getVar('BUILD_ARCH', False))
107 elif bb.data.inherits_class('crosssdk', d):
108 d.setVar('SSTATE_PKGARCH', d.expand("${BUILD_ARCH}_${SDK_ARCH}_${SDK_OS}"))
109 elif bb.data.inherits_class('cross', d):
110 d.setVar('SSTATE_PKGARCH', d.expand("${BUILD_ARCH}_${TARGET_ARCH}"))
111 elif bb.data.inherits_class('nativesdk', d):
112 d.setVar('SSTATE_PKGARCH', d.expand("${SDK_ARCH}_${SDK_OS}"))
113 elif bb.data.inherits_class('cross-canadian', d):
114 d.setVar('SSTATE_PKGARCH', d.expand("${SDK_ARCH}_${PACKAGE_ARCH}"))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500115 elif bb.data.inherits_class('allarch', d) and d.getVar("PACKAGE_ARCH") == "all":
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500116 d.setVar('SSTATE_PKGARCH', "allarch")
117 else:
118 d.setVar('SSTATE_MANMACH', d.expand("${PACKAGE_ARCH}"))
119
120 if bb.data.inherits_class('native', d) or bb.data.inherits_class('crosssdk', d) or bb.data.inherits_class('cross', d):
121 d.setVar('SSTATE_EXTRAPATH', "${NATIVELSBSTRING}/")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500122 d.setVar('BB_HASHFILENAME', "True ${SSTATE_PKGSPEC} ${SSTATE_SWSPEC}")
Brad Bishop19323692019-04-05 15:28:33 -0400123 d.setVar('SSTATE_EXTRAPATHWILDCARD', "${NATIVELSBSTRING}/")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500124
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500125 unique_tasks = sorted(set((d.getVar('SSTATETASKS') or "").split()))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500126 d.setVar('SSTATETASKS', " ".join(unique_tasks))
127 for task in unique_tasks:
128 d.prependVarFlag(task, 'prefuncs', "sstate_task_prefunc ")
129 d.appendVarFlag(task, 'postfuncs', " sstate_task_postfunc")
130}
131
132def sstate_init(task, d):
133 ss = {}
134 ss['task'] = task
135 ss['dirs'] = []
136 ss['plaindirs'] = []
137 ss['lockfiles'] = []
138 ss['lockfiles-shared'] = []
139 return ss
140
141def sstate_state_fromvars(d, task = None):
142 if task is None:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500143 task = d.getVar('BB_CURRENTTASK')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500144 if not task:
145 bb.fatal("sstate code running without task context?!")
146 task = task.replace("_setscene", "")
147
148 if task.startswith("do_"):
149 task = task[3:]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500150 inputs = (d.getVarFlag("do_" + task, 'sstate-inputdirs') or "").split()
151 outputs = (d.getVarFlag("do_" + task, 'sstate-outputdirs') or "").split()
152 plaindirs = (d.getVarFlag("do_" + task, 'sstate-plaindirs') or "").split()
153 lockfiles = (d.getVarFlag("do_" + task, 'sstate-lockfile') or "").split()
154 lockfilesshared = (d.getVarFlag("do_" + task, 'sstate-lockfile-shared') or "").split()
155 interceptfuncs = (d.getVarFlag("do_" + task, 'sstate-interceptfuncs') or "").split()
156 fixmedir = d.getVarFlag("do_" + task, 'sstate-fixmedir') or ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500157 if not task or len(inputs) != len(outputs):
158 bb.fatal("sstate variables not setup correctly?!")
159
160 if task == "populate_lic":
161 d.setVar("SSTATE_PKGSPEC", "${SSTATE_SWSPEC}")
162 d.setVar("SSTATE_EXTRAPATH", "")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500163 d.setVar('SSTATE_EXTRAPATHWILDCARD', "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500164
165 ss = sstate_init(task, d)
166 for i in range(len(inputs)):
167 sstate_add(ss, inputs[i], outputs[i], d)
168 ss['lockfiles'] = lockfiles
169 ss['lockfiles-shared'] = lockfilesshared
170 ss['plaindirs'] = plaindirs
171 ss['interceptfuncs'] = interceptfuncs
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500172 ss['fixmedir'] = fixmedir
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500173 return ss
174
175def sstate_add(ss, source, dest, d):
176 if not source.endswith("/"):
177 source = source + "/"
178 if not dest.endswith("/"):
179 dest = dest + "/"
180 source = os.path.normpath(source)
181 dest = os.path.normpath(dest)
182 srcbase = os.path.basename(source)
183 ss['dirs'].append([srcbase, source, dest])
184 return ss
185
186def sstate_install(ss, d):
187 import oe.path
188 import oe.sstatesig
189 import subprocess
190
191 sharedfiles = []
192 shareddirs = []
193 bb.utils.mkdirhier(d.expand("${SSTATE_MANIFESTS}"))
194
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500195 sstateinst = d.expand("${WORKDIR}/sstate-install-%s/" % ss['task'])
196
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500197 manifest, d2 = oe.sstatesig.sstate_get_manifest_filename(ss['task'], d)
198
199 if os.access(manifest, os.R_OK):
200 bb.fatal("Package already staged (%s)?!" % manifest)
201
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600202 d.setVar("SSTATE_INST_POSTRM", manifest + ".postrm")
203
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500204 locks = []
205 for lock in ss['lockfiles-shared']:
206 locks.append(bb.utils.lockfile(lock, True))
207 for lock in ss['lockfiles']:
208 locks.append(bb.utils.lockfile(lock))
209
210 for state in ss['dirs']:
211 bb.debug(2, "Staging files from %s to %s" % (state[1], state[2]))
212 for walkroot, dirs, files in os.walk(state[1]):
213 for file in files:
214 srcpath = os.path.join(walkroot, file)
215 dstpath = srcpath.replace(state[1], state[2])
216 #bb.debug(2, "Staging %s to %s" % (srcpath, dstpath))
217 sharedfiles.append(dstpath)
218 for dir in dirs:
219 srcdir = os.path.join(walkroot, dir)
220 dstdir = srcdir.replace(state[1], state[2])
221 #bb.debug(2, "Staging %s to %s" % (srcdir, dstdir))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500222 if os.path.islink(srcdir):
223 sharedfiles.append(dstdir)
224 continue
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500225 if not dstdir.endswith("/"):
226 dstdir = dstdir + "/"
227 shareddirs.append(dstdir)
228
229 # Check the file list for conflicts against files which already exist
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500230 whitelist = (d.getVar("SSTATE_DUPWHITELIST") or "").split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500231 match = []
232 for f in sharedfiles:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500233 if os.path.exists(f) and not os.path.islink(f):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500234 f = os.path.normpath(f)
235 realmatch = True
236 for w in whitelist:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600237 w = os.path.normpath(w)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500238 if f.startswith(w):
239 realmatch = False
240 break
241 if realmatch:
242 match.append(f)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500243 sstate_search_cmd = "grep -rlF '%s' %s --exclude=master.list | sed -e 's:^.*/::'" % (f, d.expand("${SSTATE_MANIFESTS}"))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500244 search_output = subprocess.Popen(sstate_search_cmd, shell=True, stdout=subprocess.PIPE).communicate()[0]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500245 if search_output:
246 match.append(" (matched in %s)" % search_output.decode('utf-8').rstrip())
247 else:
248 match.append(" (not matched to any task)")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500249 if match:
250 bb.error("The recipe %s is trying to install files into a shared " \
251 "area when those files already exist. Those files and their manifest " \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500252 "location are:\n %s\nPlease verify which recipe should provide the " \
253 "above files.\n\nThe build has stopped, as continuing in this scenario WILL " \
254 "break things - if not now, possibly in the future (we've seen builds fail " \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500255 "several months later). If the system knew how to recover from this " \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500256 "automatically it would, however there are several different scenarios " \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500257 "which can result in this and we don't know which one this is. It may be " \
258 "you have switched providers of something like virtual/kernel (e.g. from " \
259 "linux-yocto to linux-yocto-dev), in that case you need to execute the " \
260 "clean task for both recipes and it will resolve this error. It may be " \
261 "you changed DISTRO_FEATURES from systemd to udev or vice versa. Cleaning " \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500262 "those recipes should again resolve this error, however switching " \
263 "DISTRO_FEATURES on an existing build directory is not supported - you " \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500264 "should really clean out tmp and rebuild (reusing sstate should be safe). " \
265 "It could be the overlapping files detected are harmless in which case " \
266 "adding them to SSTATE_DUPWHITELIST may be the correct solution. It could " \
267 "also be your build is including two different conflicting versions of " \
268 "things (e.g. bluez 4 and bluez 5 and the correct solution for that would " \
269 "be to resolve the conflict. If in doubt, please ask on the mailing list, " \
270 "sharing the error and filelist above." % \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500271 (d.getVar('PN'), "\n ".join(match)))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500272 bb.fatal("If the above message is too much, the simpler version is you're advised to wipe out tmp and rebuild (reusing sstate is fine). That will likely fix things in most (but not all) cases.")
273
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500274 if ss['fixmedir'] and os.path.exists(ss['fixmedir'] + "/fixmepath.cmd"):
275 sharedfiles.append(ss['fixmedir'] + "/fixmepath.cmd")
276 sharedfiles.append(ss['fixmedir'] + "/fixmepath")
277
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500278 # Write out the manifest
279 f = open(manifest, "w")
280 for file in sharedfiles:
281 f.write(file + "\n")
282
283 # We want to ensure that directories appear at the end of the manifest
284 # so that when we test to see if they should be deleted any contents
285 # added by the task will have been removed first.
286 dirs = sorted(shareddirs, key=len)
287 # Must remove children first, which will have a longer path than the parent
288 for di in reversed(dirs):
289 f.write(di + "\n")
290 f.close()
291
292 # Append to the list of manifests for this PACKAGE_ARCH
293
294 i = d2.expand("${SSTATE_MANIFESTS}/index-${SSTATE_MANMACH}")
295 l = bb.utils.lockfile(i + ".lock")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500296 filedata = d.getVar("STAMP") + " " + d2.getVar("SSTATE_MANFILEPREFIX") + " " + d.getVar("WORKDIR") + "\n"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500297 manifests = []
298 if os.path.exists(i):
299 with open(i, "r") as f:
300 manifests = f.readlines()
301 if filedata not in manifests:
302 with open(i, "a+") as f:
303 f.write(filedata)
304 bb.utils.unlockfile(l)
305
306 # Run the actual file install
307 for state in ss['dirs']:
308 if os.path.exists(state[1]):
309 oe.path.copyhardlinktree(state[1], state[2])
310
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500311 for postinst in (d.getVar('SSTATEPOSTINSTFUNCS') or '').split():
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500312 # All hooks should run in the SSTATE_INSTDIR
313 bb.build.exec_func(postinst, d, (sstateinst,))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500314
315 for lock in locks:
316 bb.utils.unlockfile(lock)
317
318sstate_install[vardepsexclude] += "SSTATE_DUPWHITELIST STATE_MANMACH SSTATE_MANFILEPREFIX"
319sstate_install[vardeps] += "${SSTATEPOSTINSTFUNCS}"
320
321def sstate_installpkg(ss, d):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500322 from oe.gpg_sign import get_signer
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500323
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500324 sstateinst = d.expand("${WORKDIR}/sstate-install-%s/" % ss['task'])
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500325 sstatefetch = d.getVar('SSTATE_PKGNAME') + '_' + ss['task'] + ".tgz"
326 sstatepkg = d.getVar('SSTATE_PKG') + '_' + ss['task'] + ".tgz"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500327
328 if not os.path.exists(sstatepkg):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800329 pstaging_fetch(sstatefetch, d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500330
331 if not os.path.isfile(sstatepkg):
332 bb.note("Staging package %s does not exist" % sstatepkg)
333 return False
334
335 sstate_clean(ss, d)
336
337 d.setVar('SSTATE_INSTDIR', sstateinst)
338 d.setVar('SSTATE_PKG', sstatepkg)
339
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500340 if bb.utils.to_boolean(d.getVar("SSTATE_VERIFY_SIG"), False):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500341 signer = get_signer(d, 'local')
342 if not signer.verify(sstatepkg + '.sig'):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500343 bb.warn("Cannot verify signature on sstate package %s" % sstatepkg)
344
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500345 # Empty sstateinst directory, ensure its clean
346 if os.path.exists(sstateinst):
347 oe.path.remove(sstateinst)
348 bb.utils.mkdirhier(sstateinst)
349
350 sstateinst = d.getVar("SSTATE_INSTDIR")
351 d.setVar('SSTATE_FIXMEDIR', ss['fixmedir'])
352
353 for f in (d.getVar('SSTATEPREINSTFUNCS') or '').split() + ['sstate_unpack_package']:
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500354 # All hooks should run in the SSTATE_INSTDIR
355 bb.build.exec_func(f, d, (sstateinst,))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500356
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500357 return sstate_installpkgdir(ss, d)
358
359def sstate_installpkgdir(ss, d):
360 import oe.path
361 import subprocess
362
363 sstateinst = d.getVar("SSTATE_INSTDIR")
364 d.setVar('SSTATE_FIXMEDIR', ss['fixmedir'])
365
366 for f in (d.getVar('SSTATEPOSTUNPACKFUNCS') or '').split():
367 # All hooks should run in the SSTATE_INSTDIR
368 bb.build.exec_func(f, d, (sstateinst,))
369
370 def prepdir(dir):
371 # remove dir if it exists, ensure any parent directories do exist
372 if os.path.exists(dir):
373 oe.path.remove(dir)
374 bb.utils.mkdirhier(dir)
375 oe.path.remove(dir)
376
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500377 for state in ss['dirs']:
378 prepdir(state[1])
379 os.rename(sstateinst + state[0], state[1])
380 sstate_install(ss, d)
381
382 for plain in ss['plaindirs']:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500383 workdir = d.getVar('WORKDIR')
Brad Bishop977dc1a2019-02-06 16:01:43 -0500384 sharedworkdir = os.path.join(d.getVar('TMPDIR'), "work-shared")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500385 src = sstateinst + "/" + plain.replace(workdir, '')
Brad Bishop977dc1a2019-02-06 16:01:43 -0500386 if sharedworkdir in plain:
387 src = sstateinst + "/" + plain.replace(sharedworkdir, '')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500388 dest = plain
389 bb.utils.mkdirhier(src)
390 prepdir(dest)
391 os.rename(src, dest)
392
393 return True
394
395python sstate_hardcode_path_unpack () {
396 # Fixup hardcoded paths
397 #
398 # Note: The logic below must match the reverse logic in
399 # sstate_hardcode_path(d)
400 import subprocess
401
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500402 sstateinst = d.getVar('SSTATE_INSTDIR')
403 sstatefixmedir = d.getVar('SSTATE_FIXMEDIR')
404 fixmefn = sstateinst + "fixmepath"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500405 if os.path.isfile(fixmefn):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500406 staging_target = d.getVar('RECIPE_SYSROOT')
407 staging_host = d.getVar('RECIPE_SYSROOT_NATIVE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500408
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500409 if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross-canadian', d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500410 sstate_sed_cmd = "sed -i -e 's:FIXMESTAGINGDIRHOST:%s:g'" % (staging_host)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500411 elif bb.data.inherits_class('cross', d) or bb.data.inherits_class('crosssdk', d):
412 sstate_sed_cmd = "sed -i -e 's:FIXMESTAGINGDIRTARGET:%s:g; s:FIXMESTAGINGDIRHOST:%s:g'" % (staging_target, staging_host)
413 else:
414 sstate_sed_cmd = "sed -i -e 's:FIXMESTAGINGDIRTARGET:%s:g'" % (staging_target)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500415
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500416 extra_staging_fixmes = d.getVar('EXTRA_STAGING_FIXMES') or ''
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500417 for fixmevar in extra_staging_fixmes.split():
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500418 fixme_path = d.getVar(fixmevar)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500419 sstate_sed_cmd += " -e 's:FIXME_%s:%s:g'" % (fixmevar, fixme_path)
420
421 # Add sstateinst to each filename in fixmepath, use xargs to efficiently call sed
422 sstate_hardcode_cmd = "sed -e 's:^:%s:g' %s | xargs %s" % (sstateinst, fixmefn, sstate_sed_cmd)
423
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500424 # Defer do_populate_sysroot relocation command
425 if sstatefixmedir:
426 bb.utils.mkdirhier(sstatefixmedir)
427 with open(sstatefixmedir + "/fixmepath.cmd", "w") as f:
428 sstate_hardcode_cmd = sstate_hardcode_cmd.replace(fixmefn, sstatefixmedir + "/fixmepath")
429 sstate_hardcode_cmd = sstate_hardcode_cmd.replace(sstateinst, "FIXMEFINALSSTATEINST")
430 sstate_hardcode_cmd = sstate_hardcode_cmd.replace(staging_host, "FIXMEFINALSSTATEHOST")
431 sstate_hardcode_cmd = sstate_hardcode_cmd.replace(staging_target, "FIXMEFINALSSTATETARGET")
432 f.write(sstate_hardcode_cmd)
433 bb.utils.copyfile(fixmefn, sstatefixmedir + "/fixmepath")
434 return
435
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500436 bb.note("Replacing fixme paths in sstate package: %s" % (sstate_hardcode_cmd))
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500437 subprocess.check_call(sstate_hardcode_cmd, shell=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500438
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800439 # Need to remove this or we'd copy it into the target directory and may
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500440 # conflict with another writer
441 os.remove(fixmefn)
442}
443
444def sstate_clean_cachefile(ss, d):
445 import oe.path
446
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500447 sstatepkgfile = d.getVar('SSTATE_PATHSPEC') + "*_" + ss['task'] + ".tgz*"
Brad Bishopa5c52ff2018-11-23 10:55:50 +1300448 if d.getVarFlag('do_%s' % ss['task'], 'task'):
449 bb.note("Removing %s" % sstatepkgfile)
450 oe.path.remove(sstatepkgfile)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500451
452def sstate_clean_cachefiles(d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500453 for task in (d.getVar('SSTATETASKS') or "").split():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500454 ld = d.createCopy()
455 ss = sstate_state_fromvars(ld, task)
456 sstate_clean_cachefile(ss, ld)
457
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500458def sstate_clean_manifest(manifest, d, prefix=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500459 import oe.path
460
461 mfile = open(manifest)
462 entries = mfile.readlines()
463 mfile.close()
464
465 for entry in entries:
466 entry = entry.strip()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500467 if prefix and not entry.startswith("/"):
468 entry = prefix + "/" + entry
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500469 bb.debug(2, "Removing manifest: %s" % entry)
470 # We can race against another package populating directories as we're removing them
471 # so we ignore errors here.
472 try:
473 if entry.endswith("/"):
474 if os.path.islink(entry[:-1]):
475 os.remove(entry[:-1])
476 elif os.path.exists(entry) and len(os.listdir(entry)) == 0:
477 os.rmdir(entry[:-1])
478 else:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500479 os.remove(entry)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500480 except OSError:
481 pass
482
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600483 postrm = manifest + ".postrm"
484 if os.path.exists(manifest + ".postrm"):
485 import subprocess
486 os.chmod(postrm, 0o755)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500487 subprocess.check_call(postrm, shell=True)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600488 oe.path.remove(postrm)
489
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500490 oe.path.remove(manifest)
491
492def sstate_clean(ss, d):
493 import oe.path
494 import glob
495
496 d2 = d.createCopy()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500497 stamp_clean = d.getVar("STAMPCLEAN")
498 extrainf = d.getVarFlag("do_" + ss['task'], 'stamp-extra-info')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500499 if extrainf:
500 d2.setVar("SSTATE_MANMACH", extrainf)
501 wildcard_stfile = "%s.do_%s*.%s" % (stamp_clean, ss['task'], extrainf)
502 else:
503 wildcard_stfile = "%s.do_%s*" % (stamp_clean, ss['task'])
504
505 manifest = d2.expand("${SSTATE_MANFILEPREFIX}.%s" % ss['task'])
506
507 if os.path.exists(manifest):
508 locks = []
509 for lock in ss['lockfiles-shared']:
510 locks.append(bb.utils.lockfile(lock))
511 for lock in ss['lockfiles']:
512 locks.append(bb.utils.lockfile(lock))
513
514 sstate_clean_manifest(manifest, d)
515
516 for lock in locks:
517 bb.utils.unlockfile(lock)
518
519 # Remove the current and previous stamps, but keep the sigdata.
520 #
521 # The glob() matches do_task* which may match multiple tasks, for
522 # example: do_package and do_package_write_ipk, so we need to
523 # exactly match *.do_task.* and *.do_task_setscene.*
524 rm_stamp = '.do_%s.' % ss['task']
525 rm_setscene = '.do_%s_setscene.' % ss['task']
526 # For BB_SIGNATURE_HANDLER = "noop"
527 rm_nohash = ".do_%s" % ss['task']
528 for stfile in glob.glob(wildcard_stfile):
529 # Keep the sigdata
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500530 if ".sigdata." in stfile or ".sigbasedata." in stfile:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500531 continue
532 # Preserve taint files in the stamps directory
533 if stfile.endswith('.taint'):
534 continue
535 if rm_stamp in stfile or rm_setscene in stfile or \
536 stfile.endswith(rm_nohash):
537 oe.path.remove(stfile)
538
539sstate_clean[vardepsexclude] = "SSTATE_MANFILEPREFIX"
540
541CLEANFUNCS += "sstate_cleanall"
542
543python sstate_cleanall() {
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500544 bb.note("Removing shared state for package %s" % d.getVar('PN'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500545
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500546 manifest_dir = d.getVar('SSTATE_MANIFESTS')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500547 if not os.path.exists(manifest_dir):
548 return
549
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500550 tasks = d.getVar('SSTATETASKS').split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500551 for name in tasks:
552 ld = d.createCopy()
553 shared_state = sstate_state_fromvars(ld, name)
554 sstate_clean(shared_state, ld)
555}
556
557python sstate_hardcode_path () {
558 import subprocess, platform
559
560 # Need to remove hardcoded paths and fix these when we install the
561 # staging packages.
562 #
563 # Note: the logic in this function needs to match the reverse logic
564 # in sstate_installpkg(ss, d)
565
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500566 staging_target = d.getVar('RECIPE_SYSROOT')
567 staging_host = d.getVar('RECIPE_SYSROOT_NATIVE')
568 sstate_builddir = d.getVar('SSTATE_BUILDDIR')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500569
Brad Bishop316dfdd2018-06-25 12:45:53 -0400570 sstate_sed_cmd = "sed -i -e 's:%s:FIXMESTAGINGDIRHOST:g'" % staging_host
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500571 if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross-canadian', d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500572 sstate_grep_cmd = "grep -l -e '%s'" % (staging_host)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500573 elif bb.data.inherits_class('cross', d) or bb.data.inherits_class('crosssdk', d):
574 sstate_grep_cmd = "grep -l -e '%s' -e '%s'" % (staging_target, staging_host)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400575 sstate_sed_cmd += " -e 's:%s:FIXMESTAGINGDIRTARGET:g'" % staging_target
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500576 else:
Brad Bishop316dfdd2018-06-25 12:45:53 -0400577 sstate_grep_cmd = "grep -l -e '%s' -e '%s'" % (staging_target, staging_host)
578 sstate_sed_cmd += " -e 's:%s:FIXMESTAGINGDIRTARGET:g'" % staging_target
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500579
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500580 extra_staging_fixmes = d.getVar('EXTRA_STAGING_FIXMES') or ''
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500581 for fixmevar in extra_staging_fixmes.split():
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500582 fixme_path = d.getVar(fixmevar)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500583 sstate_sed_cmd += " -e 's:%s:FIXME_%s:g'" % (fixme_path, fixmevar)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500584 sstate_grep_cmd += " -e '%s'" % (fixme_path)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500585
586 fixmefn = sstate_builddir + "fixmepath"
587
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500588 sstate_scan_cmd = d.getVar('SSTATE_SCAN_CMD')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500589 sstate_filelist_cmd = "tee %s" % (fixmefn)
590
591 # fixmepath file needs relative paths, drop sstate_builddir prefix
592 sstate_filelist_relative_cmd = "sed -i -e 's:^%s::g' %s" % (sstate_builddir, fixmefn)
593
594 xargs_no_empty_run_cmd = '--no-run-if-empty'
595 if platform.system() == 'Darwin':
596 xargs_no_empty_run_cmd = ''
597
598 # Limit the fixpaths and sed operations based on the initial grep search
599 # This has the side effect of making sure the vfs cache is hot
600 sstate_hardcode_cmd = "%s | xargs %s | %s | xargs %s %s" % (sstate_scan_cmd, sstate_grep_cmd, sstate_filelist_cmd, xargs_no_empty_run_cmd, sstate_sed_cmd)
601
602 bb.note("Removing hardcoded paths from sstate package: '%s'" % (sstate_hardcode_cmd))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500603 subprocess.check_output(sstate_hardcode_cmd, shell=True, cwd=sstate_builddir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500604
605 # If the fixmefn is empty, remove it..
606 if os.stat(fixmefn).st_size == 0:
607 os.remove(fixmefn)
608 else:
609 bb.note("Replacing absolute paths in fixmepath file: '%s'" % (sstate_filelist_relative_cmd))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500610 subprocess.check_output(sstate_filelist_relative_cmd, shell=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500611}
612
613def sstate_package(ss, d):
614 import oe.path
615
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500616 tmpdir = d.getVar('TMPDIR')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500617
618 sstatebuild = d.expand("${WORKDIR}/sstate-build-%s/" % ss['task'])
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500619 sstatepkg = d.getVar('SSTATE_PKG') + '_'+ ss['task'] + ".tgz"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500620 bb.utils.remove(sstatebuild, recurse=True)
621 bb.utils.mkdirhier(sstatebuild)
622 bb.utils.mkdirhier(os.path.dirname(sstatepkg))
623 for state in ss['dirs']:
624 if not os.path.exists(state[1]):
625 continue
626 srcbase = state[0].rstrip("/").rsplit('/', 1)[0]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500627 # Find and error for absolute symlinks. We could attempt to relocate but its not
628 # clear where the symlink is relative to in this context. We could add that markup
629 # to sstate tasks but there aren't many of these so better just avoid them entirely.
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500630 for walkroot, dirs, files in os.walk(state[1]):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500631 for file in files + dirs:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500632 srcpath = os.path.join(walkroot, file)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500633 if not os.path.islink(srcpath):
634 continue
635 link = os.readlink(srcpath)
636 if not os.path.isabs(link):
637 continue
638 if not link.startswith(tmpdir):
639 continue
640 bb.error("sstate found an absolute path symlink %s pointing at %s. Please replace this with a relative link." % (srcpath, link))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500641 bb.debug(2, "Preparing tree %s for packaging at %s" % (state[1], sstatebuild + state[0]))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500642 os.rename(state[1], sstatebuild + state[0])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500643
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500644 workdir = d.getVar('WORKDIR')
Brad Bishop977dc1a2019-02-06 16:01:43 -0500645 sharedworkdir = os.path.join(d.getVar('TMPDIR'), "work-shared")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500646 for plain in ss['plaindirs']:
647 pdir = plain.replace(workdir, sstatebuild)
Brad Bishop977dc1a2019-02-06 16:01:43 -0500648 if sharedworkdir in plain:
649 pdir = plain.replace(sharedworkdir, sstatebuild)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500650 bb.utils.mkdirhier(plain)
651 bb.utils.mkdirhier(pdir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500652 os.rename(plain, pdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500653
654 d.setVar('SSTATE_BUILDDIR', sstatebuild)
655 d.setVar('SSTATE_PKG', sstatepkg)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500656 d.setVar('SSTATE_INSTDIR', sstatebuild)
657
658 if d.getVar('SSTATE_SKIP_CREATION') == '1':
659 return
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500660
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500661 for f in (d.getVar('SSTATECREATEFUNCS') or '').split() + \
Brad Bishop19323692019-04-05 15:28:33 -0400662 ['sstate_report_unihash', 'sstate_create_package', 'sstate_sign_package'] + \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500663 (d.getVar('SSTATEPOSTCREATEFUNCS') or '').split():
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500664 # All hooks should run in SSTATE_BUILDDIR.
665 bb.build.exec_func(f, d, (sstatebuild,))
666
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500667 bb.siggen.dump_this_task(sstatepkg + ".siginfo", d)
668
669 return
670
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800671def pstaging_fetch(sstatefetch, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500672 import bb.fetch2
673
674 # Only try and fetch if the user has configured a mirror
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500675 mirrors = d.getVar('SSTATE_MIRRORS')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500676 if not mirrors:
677 return
678
679 # Copy the data object and override DL_DIR and SRC_URI
680 localdata = bb.data.createCopy(d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500681
682 dldir = localdata.expand("${SSTATE_DIR}")
683 bb.utils.mkdirhier(dldir)
684
685 localdata.delVar('MIRRORS')
686 localdata.setVar('FILESPATH', dldir)
687 localdata.setVar('DL_DIR', dldir)
688 localdata.setVar('PREMIRRORS', mirrors)
689
690 # if BB_NO_NETWORK is set but we also have SSTATE_MIRROR_ALLOW_NETWORK,
691 # we'll want to allow network access for the current set of fetches.
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500692 if localdata.getVar('BB_NO_NETWORK') == "1" and localdata.getVar('SSTATE_MIRROR_ALLOW_NETWORK') == "1":
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500693 localdata.delVar('BB_NO_NETWORK')
694
695 # Try a fetch from the sstate mirror, if it fails just return and
696 # we will build the package
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600697 uris = ['file://{0};downloadfilename={0}'.format(sstatefetch),
698 'file://{0}.siginfo;downloadfilename={0}.siginfo'.format(sstatefetch)]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500699 if bb.utils.to_boolean(d.getVar("SSTATE_VERIFY_SIG"), False):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600700 uris += ['file://{0}.sig;downloadfilename={0}.sig'.format(sstatefetch)]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500701
702 for srcuri in uris:
703 localdata.setVar('SRC_URI', srcuri)
704 try:
705 fetcher = bb.fetch2.Fetch([srcuri], localdata, cache=False)
706 fetcher.download()
707
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500708 except bb.fetch2.BBFetchException:
709 break
710
711def sstate_setscene(d):
712 shared_state = sstate_state_fromvars(d)
713 accelerate = sstate_installpkg(shared_state, d)
714 if not accelerate:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600715 bb.fatal("No suitable staging package found")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500716
717python sstate_task_prefunc () {
718 shared_state = sstate_state_fromvars(d)
719 sstate_clean(shared_state, d)
720}
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500721sstate_task_prefunc[dirs] = "${WORKDIR}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500722
723python sstate_task_postfunc () {
724 shared_state = sstate_state_fromvars(d)
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500725
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500726 for intercept in shared_state['interceptfuncs']:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500727 bb.build.exec_func(intercept, d, (d.getVar("WORKDIR"),))
728
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600729 omask = os.umask(0o002)
730 if omask != 0o002:
731 bb.note("Using umask 0o002 (not %0o) for sstate packaging" % omask)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500732 sstate_package(shared_state, d)
733 os.umask(omask)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500734
735 sstateinst = d.getVar("SSTATE_INSTDIR")
736 d.setVar('SSTATE_FIXMEDIR', shared_state['fixmedir'])
737
738 sstate_installpkgdir(shared_state, d)
739
740 bb.utils.remove(d.getVar("SSTATE_BUILDDIR"), recurse=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500741}
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500742sstate_task_postfunc[dirs] = "${WORKDIR}"
743
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500744
745#
746# Shell function to generate a sstate package from a directory
747# set as SSTATE_BUILDDIR. Will be run from within SSTATE_BUILDDIR.
748#
749sstate_create_package () {
750 TFILE=`mktemp ${SSTATE_PKG}.XXXXXXXX`
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800751
752 # Use pigz if available
753 OPT="-czS"
754 if [ -x "$(command -v pigz)" ]; then
755 OPT="-I pigz -cS"
756 fi
757
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500758 # Need to handle empty directories
759 if [ "$(ls -A)" ]; then
760 set +e
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800761 tar $OPT -f $TFILE *
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500762 ret=$?
763 if [ $ret -ne 0 ] && [ $ret -ne 1 ]; then
764 exit 1
765 fi
766 set -e
767 else
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800768 tar $OPT --file=$TFILE --files-from=/dev/null
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500769 fi
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500770 chmod 0664 $TFILE
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500771 mv -f $TFILE ${SSTATE_PKG}
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500772}
773
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500774python sstate_sign_package () {
775 from oe.gpg_sign import get_signer
776
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500777 if d.getVar('SSTATE_SIG_KEY'):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500778 signer = get_signer(d, 'local')
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500779 sstate_pkg = d.getVar('SSTATE_PKG')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500780 if os.path.exists(sstate_pkg + '.sig'):
781 os.unlink(sstate_pkg + '.sig')
782 signer.detach_sign(sstate_pkg, d.getVar('SSTATE_SIG_KEY', False), None,
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500783 d.getVar('SSTATE_SIG_PASSPHRASE'), armor=False)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500784}
785
Brad Bishop19323692019-04-05 15:28:33 -0400786python sstate_report_unihash() {
787 report_unihash = getattr(bb.parse.siggen, 'report_unihash', None)
788
789 if report_unihash:
790 ss = sstate_state_fromvars(d)
791 report_unihash(os.getcwd(), ss['task'], d)
792}
793
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500794#
795# Shell function to decompress and prepare a package for installation
796# Will be run from within SSTATE_INSTDIR.
797#
798sstate_unpack_package () {
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500799 tar -xvzf ${SSTATE_PKG}
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500800 # update .siginfo atime on local/NFS mirror
801 [ -w ${SSTATE_PKG}.siginfo ] && [ -h ${SSTATE_PKG}.siginfo ] && touch -a ${SSTATE_PKG}.siginfo
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500802 # Use "! -w ||" to return true for read only files
803 [ ! -w ${SSTATE_PKG} ] || touch --no-dereference ${SSTATE_PKG}
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500804 [ ! -w ${SSTATE_PKG}.sig ] || [ ! -e ${SSTATE_PKG}.sig ] || touch --no-dereference ${SSTATE_PKG}.sig
805 [ ! -w ${SSTATE_PKG}.siginfo ] || [ ! -e ${SSTATE_PKG}.siginfo ] || touch --no-dereference ${SSTATE_PKG}.siginfo
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500806}
807
808BB_HASHCHECK_FUNCTION = "sstate_checkhashes"
809
Brad Bishop19323692019-04-05 15:28:33 -0400810def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=False, *, sq_unihash=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500811
812 ret = []
813 missed = []
814 extension = ".tgz"
815 if siginfo:
816 extension = extension + ".siginfo"
817
Brad Bishop19323692019-04-05 15:28:33 -0400818 def gethash(task):
819 if sq_unihash is not None:
820 return sq_unihash[task]
821 return sq_hash[task]
822
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500823 def getpathcomponents(task, d):
824 # Magic data from BB_HASHFILENAME
825 splithashfn = sq_hashfn[task].split(" ")
826 spec = splithashfn[1]
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500827 if splithashfn[0] == "True":
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500828 extrapath = d.getVar("NATIVELSBSTRING") + "/"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500829 else:
830 extrapath = ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500831
832 tname = sq_task[task][3:]
833
834 if tname in ["fetch", "unpack", "patch", "populate_lic", "preconfigure"] and splithashfn[2]:
835 spec = splithashfn[2]
836 extrapath = ""
837
838 return spec, extrapath, tname
839
840
841 for task in range(len(sq_fn)):
842
843 spec, extrapath, tname = getpathcomponents(task, d)
844
Brad Bishop19323692019-04-05 15:28:33 -0400845 sstatefile = d.expand("${SSTATE_DIR}/" + extrapath + generate_sstatefn(spec, gethash(task), d) + "_" + tname + extension)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500846
847 if os.path.exists(sstatefile):
848 bb.debug(2, "SState: Found valid sstate file %s" % sstatefile)
849 ret.append(task)
850 continue
851 else:
852 missed.append(task)
853 bb.debug(2, "SState: Looked for but didn't find file %s" % sstatefile)
854
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500855 mirrors = d.getVar("SSTATE_MIRRORS")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500856 if mirrors:
857 # Copy the data object and override DL_DIR and SRC_URI
858 localdata = bb.data.createCopy(d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500859
860 dldir = localdata.expand("${SSTATE_DIR}")
861 localdata.delVar('MIRRORS')
862 localdata.setVar('FILESPATH', dldir)
863 localdata.setVar('DL_DIR', dldir)
864 localdata.setVar('PREMIRRORS', mirrors)
865
866 bb.debug(2, "SState using premirror of: %s" % mirrors)
867
868 # if BB_NO_NETWORK is set but we also have SSTATE_MIRROR_ALLOW_NETWORK,
869 # we'll want to allow network access for the current set of fetches.
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500870 if localdata.getVar('BB_NO_NETWORK') == "1" and localdata.getVar('SSTATE_MIRROR_ALLOW_NETWORK') == "1":
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500871 localdata.delVar('BB_NO_NETWORK')
872
873 from bb.fetch2 import FetchConnectionCache
874 def checkstatus_init(thread_worker):
875 thread_worker.connection_cache = FetchConnectionCache()
876
877 def checkstatus_end(thread_worker):
878 thread_worker.connection_cache.close_connections()
879
880 def checkstatus(thread_worker, arg):
881 (task, sstatefile) = arg
882
883 localdata2 = bb.data.createCopy(localdata)
884 srcuri = "file://" + sstatefile
885 localdata.setVar('SRC_URI', srcuri)
886 bb.debug(2, "SState: Attempting to fetch %s" % srcuri)
887
888 try:
889 fetcher = bb.fetch2.Fetch(srcuri.split(), localdata2,
890 connection_cache=thread_worker.connection_cache)
891 fetcher.checkstatus()
892 bb.debug(2, "SState: Successful fetch test for %s" % srcuri)
893 ret.append(task)
894 if task in missed:
895 missed.remove(task)
896 except:
897 missed.append(task)
898 bb.debug(2, "SState: Unsuccessful fetch test for %s" % srcuri)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600899 pass
Brad Bishop316dfdd2018-06-25 12:45:53 -0400900 bb.event.fire(bb.event.ProcessProgress(msg, len(tasklist) - thread_worker.tasks.qsize()), d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500901
902 tasklist = []
903 for task in range(len(sq_fn)):
904 if task in ret:
905 continue
906 spec, extrapath, tname = getpathcomponents(task, d)
Brad Bishop19323692019-04-05 15:28:33 -0400907 sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(task), d) + "_" + tname + extension)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500908 tasklist.append((task, sstatefile))
909
910 if tasklist:
Brad Bishop316dfdd2018-06-25 12:45:53 -0400911 msg = "Checking sstate mirror object availability"
912 bb.event.fire(bb.event.ProcessStarted(msg, len(tasklist)), d)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600913
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500914 import multiprocessing
915 nproc = min(multiprocessing.cpu_count(), len(tasklist))
916
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600917 bb.event.enable_threadlock()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500918 pool = oe.utils.ThreadedPool(nproc, len(tasklist),
919 worker_init=checkstatus_init, worker_end=checkstatus_end)
920 for t in tasklist:
921 pool.add_task(checkstatus, t)
922 pool.start()
923 pool.wait_completion()
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600924 bb.event.disable_threadlock()
925
Brad Bishop316dfdd2018-06-25 12:45:53 -0400926 bb.event.fire(bb.event.ProcessFinished(msg), d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500927
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500928 inheritlist = d.getVar("INHERIT")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500929 if "toaster" in inheritlist:
930 evdata = {'missed': [], 'found': []};
931 for task in missed:
932 spec, extrapath, tname = getpathcomponents(task, d)
Brad Bishop19323692019-04-05 15:28:33 -0400933 sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(task), d) + "_" + tname + ".tgz")
934 evdata['missed'].append( (sq_fn[task], sq_task[task], gethash(task), sstatefile ) )
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500935 for task in ret:
936 spec, extrapath, tname = getpathcomponents(task, d)
Brad Bishop19323692019-04-05 15:28:33 -0400937 sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(task), d) + "_" + tname + ".tgz")
938 evdata['found'].append( (sq_fn[task], sq_task[task], gethash(task), sstatefile ) )
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500939 bb.event.fire(bb.event.MetadataEvent("MissedSstate", evdata), d)
940
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800941 # Print some summary statistics about the current task completion and how much sstate
942 # reuse there was. Avoid divide by zero errors.
943 total = len(sq_fn)
944 currentcount = d.getVar("BB_SETSCENE_STAMPCURRENT_COUNT") or 0
945 complete = 0
946 if currentcount:
947 complete = (len(ret) + currentcount) / (total + currentcount) * 100
948 match = 0
949 if total:
950 match = len(ret) / total * 100
951 bb.plain("Sstate summary: Wanted %d Found %d Missed %d Current %d (%d%% match, %d%% complete)" % (total, len(ret), len(missed), currentcount, match, complete))
952
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500953 if hasattr(bb.parse.siggen, "checkhashes"):
954 bb.parse.siggen.checkhashes(missed, ret, sq_fn, sq_task, sq_hash, sq_hashfn, d)
955
956 return ret
957
958BB_SETSCENE_DEPVALID = "setscene_depvalid"
959
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500960def setscene_depvalid(task, taskdependees, notneeded, d, log=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500961 # taskdependees is a dict of tasks which depend on task, each being a 3 item list of [PN, TASKNAME, FILENAME]
962 # task is included in taskdependees too
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500963 # Return - False - We need this dependency
964 # - True - We can skip this dependency
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800965 import re
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500966
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500967 def logit(msg, log):
968 if log is not None:
969 log.append(msg)
970 else:
971 bb.debug(2, msg)
972
973 logit("Considering setscene task: %s" % (str(taskdependees[task])), log)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500974
975 def isNativeCross(x):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500976 return x.endswith("-native") or "-cross-" in x or "-crosssdk" in x or x.endswith("-cross")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500977
978 # We only need to trigger populate_lic through direct dependencies
979 if taskdependees[task][1] == "do_populate_lic":
980 return True
981
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500982 # stash_locale and gcc_stash_builddir are never needed as a dependency for built objects
983 if taskdependees[task][1] == "do_stash_locale" or taskdependees[task][1] == "do_gcc_stash_builddir":
984 return True
985
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500986 # We only need to trigger packagedata through direct dependencies
987 # but need to preserve packagedata on packagedata links
988 if taskdependees[task][1] == "do_packagedata":
989 for dep in taskdependees:
990 if taskdependees[dep][1] == "do_packagedata":
991 return False
992 return True
993
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500994 for dep in taskdependees:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500995 logit(" considering dependency: %s" % (str(taskdependees[dep])), log)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500996 if task == dep:
997 continue
998 if dep in notneeded:
999 continue
1000 # do_package_write_* and do_package doesn't need do_package
1001 if taskdependees[task][1] == "do_package" and taskdependees[dep][1] in ['do_package', 'do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm', 'do_packagedata', 'do_package_qa']:
1002 continue
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001003 # do_package_write_* need do_populate_sysroot as they're mainly postinstall dependencies
1004 if taskdependees[task][1] == "do_populate_sysroot" and taskdependees[dep][1] in ['do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm']:
1005 return False
1006 # do_package/packagedata/package_qa don't need do_populate_sysroot
1007 if taskdependees[task][1] == "do_populate_sysroot" and taskdependees[dep][1] in ['do_package', 'do_packagedata', 'do_package_qa']:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001008 continue
1009 # Native/Cross packages don't exist and are noexec anyway
1010 if isNativeCross(taskdependees[dep][0]) and taskdependees[dep][1] in ['do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm', 'do_packagedata', 'do_package', 'do_package_qa']:
1011 continue
1012
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001013 # This is due to the [depends] in useradd.bbclass complicating matters
1014 # The logic *is* reversed here due to the way hard setscene dependencies are injected
1015 if (taskdependees[task][1] == 'do_package' or taskdependees[task][1] == 'do_populate_sysroot') and taskdependees[dep][0].endswith(('shadow-native', 'shadow-sysroot', 'base-passwd', 'pseudo-native')) and taskdependees[dep][1] == 'do_populate_sysroot':
1016 continue
1017
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001018 # Consider sysroot depending on sysroot tasks
1019 if taskdependees[task][1] == 'do_populate_sysroot' and taskdependees[dep][1] == 'do_populate_sysroot':
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001020 # Allow excluding certain recursive dependencies. If a recipe needs it should add a
1021 # specific dependency itself, rather than relying on one of its dependees to pull
1022 # them in.
1023 # See also http://lists.openembedded.org/pipermail/openembedded-core/2018-January/146324.html
1024 not_needed = False
1025 excludedeps = d.getVar('_SSTATE_EXCLUDEDEPS_SYSROOT')
1026 if excludedeps is None:
1027 # Cache the regular expressions for speed
1028 excludedeps = []
1029 for excl in (d.getVar('SSTATE_EXCLUDEDEPS_SYSROOT') or "").split():
1030 excludedeps.append((re.compile(excl.split('->', 1)[0]), re.compile(excl.split('->', 1)[1])))
1031 d.setVar('_SSTATE_EXCLUDEDEPS_SYSROOT', excludedeps)
1032 for excl in excludedeps:
1033 if excl[0].match(taskdependees[dep][0]):
1034 if excl[1].match(taskdependees[task][0]):
1035 not_needed = True
1036 break
1037 if not_needed:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001038 continue
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001039 # For meta-extsdk-toolchain we want all sysroot dependencies
1040 if taskdependees[dep][0] == 'meta-extsdk-toolchain':
1041 return False
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001042 # Native/Cross populate_sysroot need their dependencies
1043 if isNativeCross(taskdependees[task][0]) and isNativeCross(taskdependees[dep][0]):
1044 return False
1045 # Target populate_sysroot depended on by cross tools need to be installed
1046 if isNativeCross(taskdependees[dep][0]):
1047 return False
1048 # Native/cross tools depended upon by target sysroot are not needed
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001049 # Add an exception for shadow-native as required by useradd.bbclass
1050 if isNativeCross(taskdependees[task][0]) and taskdependees[task][0] != 'shadow-native':
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001051 continue
1052 # Target populate_sysroot need their dependencies
1053 return False
1054
1055 if taskdependees[task][1] == 'do_shared_workdir':
1056 continue
1057
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001058 if taskdependees[dep][1] == "do_populate_lic":
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001059 continue
1060
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001061
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001062 # Safe fallthrough default
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001063 logit(" Default setscene dependency fall through due to dependency: %s" % (str(taskdependees[dep])), log)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001064 return False
1065 return True
1066
1067addhandler sstate_eventhandler
1068sstate_eventhandler[eventmask] = "bb.build.TaskSucceeded"
1069python sstate_eventhandler() {
1070 d = e.data
1071 # When we write an sstate package we rewrite the SSTATE_PKG
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001072 spkg = d.getVar('SSTATE_PKG')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001073 if not spkg.endswith(".tgz"):
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001074 taskname = d.getVar("BB_RUNTASK")[3:]
1075 spec = d.getVar('SSTATE_PKGSPEC')
1076 swspec = d.getVar('SSTATE_SWSPEC')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001077 if taskname in ["fetch", "unpack", "patch", "populate_lic", "preconfigure"] and swspec:
1078 d.setVar("SSTATE_PKGSPEC", "${SSTATE_SWSPEC}")
1079 d.setVar("SSTATE_EXTRAPATH", "")
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001080 sstatepkg = d.getVar('SSTATE_PKG')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001081 bb.siggen.dump_this_task(sstatepkg + '_' + taskname + ".tgz" ".siginfo", d)
1082}
1083
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001084SSTATE_PRUNE_OBSOLETEWORKDIR ?= "1"
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001085
1086# Event handler which removes manifests and stamps file for
1087# recipes which are no longer reachable in a build where they
1088# once were.
1089# Also optionally removes the workdir of those tasks/recipes
1090#
1091addhandler sstate_eventhandler2
1092sstate_eventhandler2[eventmask] = "bb.event.ReachableStamps"
1093python sstate_eventhandler2() {
1094 import glob
1095 d = e.data
1096 stamps = e.stamps.values()
1097 removeworkdir = (d.getVar("SSTATE_PRUNE_OBSOLETEWORKDIR", False) == "1")
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001098 preservestampfile = d.expand('${SSTATE_MANIFESTS}/preserve-stamps')
1099 preservestamps = []
1100 if os.path.exists(preservestampfile):
1101 with open(preservestampfile, 'r') as f:
1102 preservestamps = f.readlines()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001103 seen = []
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001104
1105 # The machine index contains all the stamps this machine has ever seen in this build directory.
1106 # We should only remove things which this machine once accessed but no longer does.
1107 machineindex = set()
1108 bb.utils.mkdirhier(d.expand("${SSTATE_MANIFESTS}"))
1109 mi = d.expand("${SSTATE_MANIFESTS}/index-machine-${MACHINE}")
1110 if os.path.exists(mi):
1111 with open(mi, "r") as f:
1112 machineindex = set(line.strip() for line in f.readlines())
1113
Brad Bishop316dfdd2018-06-25 12:45:53 -04001114 for a in sorted(list(set(d.getVar("SSTATE_ARCHS").split()))):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001115 toremove = []
1116 i = d.expand("${SSTATE_MANIFESTS}/index-" + a)
1117 if not os.path.exists(i):
1118 continue
1119 with open(i, "r") as f:
1120 lines = f.readlines()
1121 for l in lines:
Brad Bishop19323692019-04-05 15:28:33 -04001122 try:
1123 (stamp, manifest, workdir) = l.split()
1124 if stamp not in stamps and stamp not in preservestamps and stamp in machineindex:
1125 toremove.append(l)
1126 if stamp not in seen:
1127 bb.debug(2, "Stamp %s is not reachable, removing related manifests" % stamp)
1128 seen.append(stamp)
1129 except ValueError:
1130 bb.fatal("Invalid line '%s' in sstate manifest '%s'" % (l, i))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001131
1132 if toremove:
Brad Bishop316dfdd2018-06-25 12:45:53 -04001133 msg = "Removing %d recipes from the %s sysroot" % (len(toremove), a)
1134 bb.event.fire(bb.event.ProcessStarted(msg, len(toremove)), d)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001135
Brad Bishop316dfdd2018-06-25 12:45:53 -04001136 removed = 0
1137 for r in toremove:
1138 (stamp, manifest, workdir) = r.split()
1139 for m in glob.glob(manifest + ".*"):
1140 if m.endswith(".postrm"):
1141 continue
1142 sstate_clean_manifest(m, d)
1143 bb.utils.remove(stamp + "*")
1144 if removeworkdir:
1145 bb.utils.remove(workdir, recurse = True)
1146 lines.remove(r)
1147 removed = removed + 1
1148 bb.event.fire(bb.event.ProcessProgress(msg, removed), d)
1149
1150 bb.event.fire(bb.event.ProcessFinished(msg), d)
1151
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001152 with open(i, "w") as f:
1153 for l in lines:
1154 f.write(l)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001155 machineindex |= set(stamps)
1156 with open(mi, "w") as f:
1157 for l in machineindex:
1158 f.write(l + "\n")
1159
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001160 if preservestamps:
1161 os.remove(preservestampfile)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001162}