blob: b47b9c23bf21685978ab5cefba64d88840cd1a90 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001SSTATE_VERSION = "3"
2
3SSTATE_MANIFESTS ?= "${TMPDIR}/sstate-control"
4SSTATE_MANFILEPREFIX = "${SSTATE_MANIFESTS}/manifest-${SSTATE_MANMACH}-${PN}"
5
6def generate_sstatefn(spec, hash, d):
7 if not hash:
8 hash = "INVALID"
9 return hash[:2] + "/" + spec + hash
10
11SSTATE_PKGARCH = "${PACKAGE_ARCH}"
12SSTATE_PKGSPEC = "sstate:${PN}:${PACKAGE_ARCH}${TARGET_VENDOR}-${TARGET_OS}:${PV}:${PR}:${SSTATE_PKGARCH}:${SSTATE_VERSION}:"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050013SSTATE_SWSPEC = "sstate:${PN}::${PV}:${PR}::${SSTATE_VERSION}:"
Brad Bishop19323692019-04-05 15:28:33 -040014SSTATE_PKGNAME = "${SSTATE_EXTRAPATH}${@generate_sstatefn(d.getVar('SSTATE_PKGSPEC'), d.getVar('BB_UNIHASH'), d)}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050015SSTATE_PKG = "${SSTATE_DIR}/${SSTATE_PKGNAME}"
16SSTATE_EXTRAPATH = ""
17SSTATE_EXTRAPATHWILDCARD = ""
18SSTATE_PATHSPEC = "${SSTATE_DIR}/${SSTATE_EXTRAPATHWILDCARD}*/${SSTATE_PKGSPEC}"
19
Patrick Williamsc0f7c042017-02-23 20:41:17 -060020# explicitly make PV to depend on evaluated value of PV variable
21PV[vardepvalue] = "${PV}"
22
Patrick Williamsc124f4f2015-09-15 14:41:29 -050023# We don't want the sstate to depend on things like the distro string
24# of the system, we let the sstate paths take care of this.
25SSTATE_EXTRAPATH[vardepvalue] = ""
Brad Bishop19323692019-04-05 15:28:33 -040026SSTATE_EXTRAPATHWILDCARD[vardepvalue] = ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -050027
28# For multilib rpm the allarch packagegroup files can overwrite (in theory they're identical)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080029SSTATE_DUPWHITELIST = "${DEPLOY_DIR}/licenses/"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050030# Avoid docbook/sgml catalog warnings for now
31SSTATE_DUPWHITELIST += "${STAGING_ETCDIR_NATIVE}/sgml ${STAGING_DATADIR_NATIVE}/sgml"
Brad Bishop316dfdd2018-06-25 12:45:53 -040032# sdk-provides-dummy-nativesdk and nativesdk-buildtools-perl-dummy overlap for different SDKMACHINE
33SSTATE_DUPWHITELIST += "${DEPLOY_DIR_RPM}/sdk_provides_dummy_nativesdk/ ${DEPLOY_DIR_IPK}/sdk-provides-dummy-nativesdk/"
34SSTATE_DUPWHITELIST += "${DEPLOY_DIR_RPM}/buildtools_dummy_nativesdk/ ${DEPLOY_DIR_IPK}/buildtools-dummy-nativesdk/"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080035# target-sdk-provides-dummy overlaps that allarch is disabled when multilib is used
36SSTATE_DUPWHITELIST += "${COMPONENTS_DIR}/sdk-provides-dummy-target/ ${DEPLOY_DIR_RPM}/sdk_provides_dummy_target/ ${DEPLOY_DIR_IPK}/sdk-provides-dummy-target/"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050037# Archive the sources for many architectures in one deploy folder
38SSTATE_DUPWHITELIST += "${DEPLOY_DIR_SRC}"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080039# ovmf/grub-efi/systemd-boot/intel-microcode multilib recipes can generate identical overlapping files
40SSTATE_DUPWHITELIST += "${DEPLOY_DIR_IMAGE}/ovmf"
41SSTATE_DUPWHITELIST += "${DEPLOY_DIR_IMAGE}/grub-efi"
42SSTATE_DUPWHITELIST += "${DEPLOY_DIR_IMAGE}/systemd-boot"
43SSTATE_DUPWHITELIST += "${DEPLOY_DIR_IMAGE}/microcode"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050044
Brad Bishop6e60e8b2018-02-01 10:27:11 -050045SSTATE_SCAN_FILES ?= "*.la *-config *_config postinst-*"
46SSTATE_SCAN_CMD ??= 'find ${SSTATE_BUILDDIR} \( -name "${@"\" -o -name \"".join(d.getVar("SSTATE_SCAN_FILES").split())}" \) -type f'
47SSTATE_SCAN_CMD_NATIVE ??= 'grep -Irl -e ${RECIPE_SYSROOT} -e ${RECIPE_SYSROOT_NATIVE} -e ${HOSTTOOLS_DIR} ${SSTATE_BUILDDIR}'
Patrick Williamsc124f4f2015-09-15 14:41:29 -050048
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050049BB_HASHFILENAME = "False ${SSTATE_PKGSPEC} ${SSTATE_SWSPEC}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050050
51SSTATE_ARCHS = " \
52 ${BUILD_ARCH} \
53 ${BUILD_ARCH}_${SDK_ARCH}_${SDK_OS} \
54 ${BUILD_ARCH}_${TARGET_ARCH} \
55 ${SDK_ARCH}_${SDK_OS} \
56 ${SDK_ARCH}_${PACKAGE_ARCH} \
57 allarch \
58 ${PACKAGE_ARCH} \
Brad Bishop316dfdd2018-06-25 12:45:53 -040059 ${PACKAGE_EXTRA_ARCHS} \
60 ${MACHINE_ARCH}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050061
62SSTATE_MANMACH ?= "${SSTATE_PKGARCH}"
63
64SSTATECREATEFUNCS = "sstate_hardcode_path"
Brad Bishop19323692019-04-05 15:28:33 -040065SSTATECREATEFUNCS[vardeps] = "SSTATE_SCAN_FILES"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050066SSTATEPOSTCREATEFUNCS = ""
67SSTATEPREINSTFUNCS = ""
68SSTATEPOSTUNPACKFUNCS = "sstate_hardcode_path_unpack"
69SSTATEPOSTINSTFUNCS = ""
Brad Bishop6e60e8b2018-02-01 10:27:11 -050070EXTRA_STAGING_FIXMES ?= "HOSTTOOLS_DIR"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050071
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050072# Check whether sstate exists for tasks that support sstate and are in the
73# locked signatures file.
74SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK ?= 'error'
75
76# Check whether the task's computed hash matches the task's hash in the
77# locked signatures file.
78SIGGEN_LOCKEDSIGS_TASKSIG_CHECK ?= "error"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050079
80# The GnuPG key ID and passphrase to use to sign sstate archives (or unset to
81# not sign)
82SSTATE_SIG_KEY ?= ""
83SSTATE_SIG_PASSPHRASE ?= ""
84# Whether to verify the GnUPG signatures when extracting sstate archives
85SSTATE_VERIFY_SIG ?= "0"
86
Brad Bishop19323692019-04-05 15:28:33 -040087SSTATE_HASHEQUIV_METHOD ?= "oe.sstatesig.OEOuthashBasic"
88SSTATE_HASHEQUIV_METHOD[doc] = "The fully-qualified function used to calculate \
89 the output hash for a task, which in turn is used to determine equivalency. \
90 "
91
Brad Bishop19323692019-04-05 15:28:33 -040092SSTATE_HASHEQUIV_REPORT_TASKDATA ?= "0"
93SSTATE_HASHEQUIV_REPORT_TASKDATA[doc] = "Report additional useful data to the \
94 hash equivalency server, such as PN, PV, taskname, etc. This information \
95 is very useful for developers looking at task data, but may leak sensitive \
96 data if the equivalence server is public. \
97 "
98
Patrick Williamsc124f4f2015-09-15 14:41:29 -050099python () {
100 if bb.data.inherits_class('native', d):
101 d.setVar('SSTATE_PKGARCH', d.getVar('BUILD_ARCH', False))
102 elif bb.data.inherits_class('crosssdk', d):
103 d.setVar('SSTATE_PKGARCH', d.expand("${BUILD_ARCH}_${SDK_ARCH}_${SDK_OS}"))
104 elif bb.data.inherits_class('cross', d):
105 d.setVar('SSTATE_PKGARCH', d.expand("${BUILD_ARCH}_${TARGET_ARCH}"))
106 elif bb.data.inherits_class('nativesdk', d):
107 d.setVar('SSTATE_PKGARCH', d.expand("${SDK_ARCH}_${SDK_OS}"))
108 elif bb.data.inherits_class('cross-canadian', d):
109 d.setVar('SSTATE_PKGARCH', d.expand("${SDK_ARCH}_${PACKAGE_ARCH}"))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500110 elif bb.data.inherits_class('allarch', d) and d.getVar("PACKAGE_ARCH") == "all":
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500111 d.setVar('SSTATE_PKGARCH', "allarch")
112 else:
113 d.setVar('SSTATE_MANMACH', d.expand("${PACKAGE_ARCH}"))
114
115 if bb.data.inherits_class('native', d) or bb.data.inherits_class('crosssdk', d) or bb.data.inherits_class('cross', d):
116 d.setVar('SSTATE_EXTRAPATH', "${NATIVELSBSTRING}/")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500117 d.setVar('BB_HASHFILENAME', "True ${SSTATE_PKGSPEC} ${SSTATE_SWSPEC}")
Brad Bishop19323692019-04-05 15:28:33 -0400118 d.setVar('SSTATE_EXTRAPATHWILDCARD', "${NATIVELSBSTRING}/")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500119
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500120 unique_tasks = sorted(set((d.getVar('SSTATETASKS') or "").split()))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500121 d.setVar('SSTATETASKS', " ".join(unique_tasks))
122 for task in unique_tasks:
123 d.prependVarFlag(task, 'prefuncs', "sstate_task_prefunc ")
124 d.appendVarFlag(task, 'postfuncs', " sstate_task_postfunc")
125}
126
127def sstate_init(task, d):
128 ss = {}
129 ss['task'] = task
130 ss['dirs'] = []
131 ss['plaindirs'] = []
132 ss['lockfiles'] = []
133 ss['lockfiles-shared'] = []
134 return ss
135
136def sstate_state_fromvars(d, task = None):
137 if task is None:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500138 task = d.getVar('BB_CURRENTTASK')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500139 if not task:
140 bb.fatal("sstate code running without task context?!")
141 task = task.replace("_setscene", "")
142
143 if task.startswith("do_"):
144 task = task[3:]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500145 inputs = (d.getVarFlag("do_" + task, 'sstate-inputdirs') or "").split()
146 outputs = (d.getVarFlag("do_" + task, 'sstate-outputdirs') or "").split()
147 plaindirs = (d.getVarFlag("do_" + task, 'sstate-plaindirs') or "").split()
148 lockfiles = (d.getVarFlag("do_" + task, 'sstate-lockfile') or "").split()
149 lockfilesshared = (d.getVarFlag("do_" + task, 'sstate-lockfile-shared') or "").split()
150 interceptfuncs = (d.getVarFlag("do_" + task, 'sstate-interceptfuncs') or "").split()
151 fixmedir = d.getVarFlag("do_" + task, 'sstate-fixmedir') or ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500152 if not task or len(inputs) != len(outputs):
153 bb.fatal("sstate variables not setup correctly?!")
154
155 if task == "populate_lic":
156 d.setVar("SSTATE_PKGSPEC", "${SSTATE_SWSPEC}")
157 d.setVar("SSTATE_EXTRAPATH", "")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500158 d.setVar('SSTATE_EXTRAPATHWILDCARD', "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500159
160 ss = sstate_init(task, d)
161 for i in range(len(inputs)):
162 sstate_add(ss, inputs[i], outputs[i], d)
163 ss['lockfiles'] = lockfiles
164 ss['lockfiles-shared'] = lockfilesshared
165 ss['plaindirs'] = plaindirs
166 ss['interceptfuncs'] = interceptfuncs
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500167 ss['fixmedir'] = fixmedir
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500168 return ss
169
170def sstate_add(ss, source, dest, d):
171 if not source.endswith("/"):
172 source = source + "/"
173 if not dest.endswith("/"):
174 dest = dest + "/"
175 source = os.path.normpath(source)
176 dest = os.path.normpath(dest)
177 srcbase = os.path.basename(source)
178 ss['dirs'].append([srcbase, source, dest])
179 return ss
180
181def sstate_install(ss, d):
182 import oe.path
183 import oe.sstatesig
184 import subprocess
185
186 sharedfiles = []
187 shareddirs = []
188 bb.utils.mkdirhier(d.expand("${SSTATE_MANIFESTS}"))
189
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500190 sstateinst = d.expand("${WORKDIR}/sstate-install-%s/" % ss['task'])
191
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500192 manifest, d2 = oe.sstatesig.sstate_get_manifest_filename(ss['task'], d)
193
194 if os.access(manifest, os.R_OK):
195 bb.fatal("Package already staged (%s)?!" % manifest)
196
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600197 d.setVar("SSTATE_INST_POSTRM", manifest + ".postrm")
198
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500199 locks = []
200 for lock in ss['lockfiles-shared']:
201 locks.append(bb.utils.lockfile(lock, True))
202 for lock in ss['lockfiles']:
203 locks.append(bb.utils.lockfile(lock))
204
205 for state in ss['dirs']:
206 bb.debug(2, "Staging files from %s to %s" % (state[1], state[2]))
207 for walkroot, dirs, files in os.walk(state[1]):
208 for file in files:
209 srcpath = os.path.join(walkroot, file)
210 dstpath = srcpath.replace(state[1], state[2])
211 #bb.debug(2, "Staging %s to %s" % (srcpath, dstpath))
212 sharedfiles.append(dstpath)
213 for dir in dirs:
214 srcdir = os.path.join(walkroot, dir)
215 dstdir = srcdir.replace(state[1], state[2])
216 #bb.debug(2, "Staging %s to %s" % (srcdir, dstdir))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500217 if os.path.islink(srcdir):
218 sharedfiles.append(dstdir)
219 continue
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500220 if not dstdir.endswith("/"):
221 dstdir = dstdir + "/"
222 shareddirs.append(dstdir)
223
224 # Check the file list for conflicts against files which already exist
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500225 whitelist = (d.getVar("SSTATE_DUPWHITELIST") or "").split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500226 match = []
227 for f in sharedfiles:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500228 if os.path.exists(f) and not os.path.islink(f):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500229 f = os.path.normpath(f)
230 realmatch = True
231 for w in whitelist:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600232 w = os.path.normpath(w)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500233 if f.startswith(w):
234 realmatch = False
235 break
236 if realmatch:
237 match.append(f)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500238 sstate_search_cmd = "grep -rlF '%s' %s --exclude=master.list | sed -e 's:^.*/::'" % (f, d.expand("${SSTATE_MANIFESTS}"))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500239 search_output = subprocess.Popen(sstate_search_cmd, shell=True, stdout=subprocess.PIPE).communicate()[0]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500240 if search_output:
241 match.append(" (matched in %s)" % search_output.decode('utf-8').rstrip())
242 else:
243 match.append(" (not matched to any task)")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500244 if match:
245 bb.error("The recipe %s is trying to install files into a shared " \
246 "area when those files already exist. Those files and their manifest " \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500247 "location are:\n %s\nPlease verify which recipe should provide the " \
248 "above files.\n\nThe build has stopped, as continuing in this scenario WILL " \
249 "break things - if not now, possibly in the future (we've seen builds fail " \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500250 "several months later). If the system knew how to recover from this " \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500251 "automatically it would, however there are several different scenarios " \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500252 "which can result in this and we don't know which one this is. It may be " \
253 "you have switched providers of something like virtual/kernel (e.g. from " \
254 "linux-yocto to linux-yocto-dev), in that case you need to execute the " \
255 "clean task for both recipes and it will resolve this error. It may be " \
256 "you changed DISTRO_FEATURES from systemd to udev or vice versa. Cleaning " \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500257 "those recipes should again resolve this error, however switching " \
258 "DISTRO_FEATURES on an existing build directory is not supported - you " \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500259 "should really clean out tmp and rebuild (reusing sstate should be safe). " \
260 "It could be the overlapping files detected are harmless in which case " \
261 "adding them to SSTATE_DUPWHITELIST may be the correct solution. It could " \
262 "also be your build is including two different conflicting versions of " \
263 "things (e.g. bluez 4 and bluez 5 and the correct solution for that would " \
264 "be to resolve the conflict. If in doubt, please ask on the mailing list, " \
265 "sharing the error and filelist above." % \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500266 (d.getVar('PN'), "\n ".join(match)))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500267 bb.fatal("If the above message is too much, the simpler version is you're advised to wipe out tmp and rebuild (reusing sstate is fine). That will likely fix things in most (but not all) cases.")
268
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500269 if ss['fixmedir'] and os.path.exists(ss['fixmedir'] + "/fixmepath.cmd"):
270 sharedfiles.append(ss['fixmedir'] + "/fixmepath.cmd")
271 sharedfiles.append(ss['fixmedir'] + "/fixmepath")
272
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500273 # Write out the manifest
274 f = open(manifest, "w")
275 for file in sharedfiles:
276 f.write(file + "\n")
277
278 # We want to ensure that directories appear at the end of the manifest
279 # so that when we test to see if they should be deleted any contents
280 # added by the task will have been removed first.
281 dirs = sorted(shareddirs, key=len)
282 # Must remove children first, which will have a longer path than the parent
283 for di in reversed(dirs):
284 f.write(di + "\n")
285 f.close()
286
287 # Append to the list of manifests for this PACKAGE_ARCH
288
289 i = d2.expand("${SSTATE_MANIFESTS}/index-${SSTATE_MANMACH}")
290 l = bb.utils.lockfile(i + ".lock")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500291 filedata = d.getVar("STAMP") + " " + d2.getVar("SSTATE_MANFILEPREFIX") + " " + d.getVar("WORKDIR") + "\n"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500292 manifests = []
293 if os.path.exists(i):
294 with open(i, "r") as f:
295 manifests = f.readlines()
296 if filedata not in manifests:
297 with open(i, "a+") as f:
298 f.write(filedata)
299 bb.utils.unlockfile(l)
300
301 # Run the actual file install
302 for state in ss['dirs']:
303 if os.path.exists(state[1]):
304 oe.path.copyhardlinktree(state[1], state[2])
305
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500306 for postinst in (d.getVar('SSTATEPOSTINSTFUNCS') or '').split():
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500307 # All hooks should run in the SSTATE_INSTDIR
308 bb.build.exec_func(postinst, d, (sstateinst,))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500309
310 for lock in locks:
311 bb.utils.unlockfile(lock)
312
313sstate_install[vardepsexclude] += "SSTATE_DUPWHITELIST STATE_MANMACH SSTATE_MANFILEPREFIX"
314sstate_install[vardeps] += "${SSTATEPOSTINSTFUNCS}"
315
316def sstate_installpkg(ss, d):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500317 from oe.gpg_sign import get_signer
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500318
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500319 sstateinst = d.expand("${WORKDIR}/sstate-install-%s/" % ss['task'])
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500320 sstatefetch = d.getVar('SSTATE_PKGNAME') + '_' + ss['task'] + ".tgz"
321 sstatepkg = d.getVar('SSTATE_PKG') + '_' + ss['task'] + ".tgz"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500322
323 if not os.path.exists(sstatepkg):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800324 pstaging_fetch(sstatefetch, d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500325
326 if not os.path.isfile(sstatepkg):
Brad Bishop08902b02019-08-20 09:16:51 -0400327 bb.note("Sstate package %s does not exist" % sstatepkg)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500328 return False
329
330 sstate_clean(ss, d)
331
332 d.setVar('SSTATE_INSTDIR', sstateinst)
333 d.setVar('SSTATE_PKG', sstatepkg)
334
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500335 if bb.utils.to_boolean(d.getVar("SSTATE_VERIFY_SIG"), False):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500336 signer = get_signer(d, 'local')
337 if not signer.verify(sstatepkg + '.sig'):
Brad Bishop08902b02019-08-20 09:16:51 -0400338 bb.warn("Cannot verify signature on sstate package %s, skipping acceleration..." % sstatepkg)
339 return False
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500340
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500341 # Empty sstateinst directory, ensure its clean
342 if os.path.exists(sstateinst):
343 oe.path.remove(sstateinst)
344 bb.utils.mkdirhier(sstateinst)
345
346 sstateinst = d.getVar("SSTATE_INSTDIR")
347 d.setVar('SSTATE_FIXMEDIR', ss['fixmedir'])
348
349 for f in (d.getVar('SSTATEPREINSTFUNCS') or '').split() + ['sstate_unpack_package']:
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500350 # All hooks should run in the SSTATE_INSTDIR
351 bb.build.exec_func(f, d, (sstateinst,))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500352
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500353 return sstate_installpkgdir(ss, d)
354
355def sstate_installpkgdir(ss, d):
356 import oe.path
357 import subprocess
358
359 sstateinst = d.getVar("SSTATE_INSTDIR")
360 d.setVar('SSTATE_FIXMEDIR', ss['fixmedir'])
361
362 for f in (d.getVar('SSTATEPOSTUNPACKFUNCS') or '').split():
363 # All hooks should run in the SSTATE_INSTDIR
364 bb.build.exec_func(f, d, (sstateinst,))
365
366 def prepdir(dir):
367 # remove dir if it exists, ensure any parent directories do exist
368 if os.path.exists(dir):
369 oe.path.remove(dir)
370 bb.utils.mkdirhier(dir)
371 oe.path.remove(dir)
372
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500373 for state in ss['dirs']:
374 prepdir(state[1])
375 os.rename(sstateinst + state[0], state[1])
376 sstate_install(ss, d)
377
378 for plain in ss['plaindirs']:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500379 workdir = d.getVar('WORKDIR')
Brad Bishop977dc1a2019-02-06 16:01:43 -0500380 sharedworkdir = os.path.join(d.getVar('TMPDIR'), "work-shared")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500381 src = sstateinst + "/" + plain.replace(workdir, '')
Brad Bishop977dc1a2019-02-06 16:01:43 -0500382 if sharedworkdir in plain:
383 src = sstateinst + "/" + plain.replace(sharedworkdir, '')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500384 dest = plain
385 bb.utils.mkdirhier(src)
386 prepdir(dest)
387 os.rename(src, dest)
388
389 return True
390
391python sstate_hardcode_path_unpack () {
392 # Fixup hardcoded paths
393 #
394 # Note: The logic below must match the reverse logic in
395 # sstate_hardcode_path(d)
396 import subprocess
397
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500398 sstateinst = d.getVar('SSTATE_INSTDIR')
399 sstatefixmedir = d.getVar('SSTATE_FIXMEDIR')
400 fixmefn = sstateinst + "fixmepath"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500401 if os.path.isfile(fixmefn):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500402 staging_target = d.getVar('RECIPE_SYSROOT')
403 staging_host = d.getVar('RECIPE_SYSROOT_NATIVE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500404
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500405 if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross-canadian', d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500406 sstate_sed_cmd = "sed -i -e 's:FIXMESTAGINGDIRHOST:%s:g'" % (staging_host)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500407 elif bb.data.inherits_class('cross', d) or bb.data.inherits_class('crosssdk', d):
408 sstate_sed_cmd = "sed -i -e 's:FIXMESTAGINGDIRTARGET:%s:g; s:FIXMESTAGINGDIRHOST:%s:g'" % (staging_target, staging_host)
409 else:
410 sstate_sed_cmd = "sed -i -e 's:FIXMESTAGINGDIRTARGET:%s:g'" % (staging_target)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500411
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500412 extra_staging_fixmes = d.getVar('EXTRA_STAGING_FIXMES') or ''
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500413 for fixmevar in extra_staging_fixmes.split():
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500414 fixme_path = d.getVar(fixmevar)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500415 sstate_sed_cmd += " -e 's:FIXME_%s:%s:g'" % (fixmevar, fixme_path)
416
417 # Add sstateinst to each filename in fixmepath, use xargs to efficiently call sed
418 sstate_hardcode_cmd = "sed -e 's:^:%s:g' %s | xargs %s" % (sstateinst, fixmefn, sstate_sed_cmd)
419
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500420 # Defer do_populate_sysroot relocation command
421 if sstatefixmedir:
422 bb.utils.mkdirhier(sstatefixmedir)
423 with open(sstatefixmedir + "/fixmepath.cmd", "w") as f:
424 sstate_hardcode_cmd = sstate_hardcode_cmd.replace(fixmefn, sstatefixmedir + "/fixmepath")
425 sstate_hardcode_cmd = sstate_hardcode_cmd.replace(sstateinst, "FIXMEFINALSSTATEINST")
426 sstate_hardcode_cmd = sstate_hardcode_cmd.replace(staging_host, "FIXMEFINALSSTATEHOST")
427 sstate_hardcode_cmd = sstate_hardcode_cmd.replace(staging_target, "FIXMEFINALSSTATETARGET")
428 f.write(sstate_hardcode_cmd)
429 bb.utils.copyfile(fixmefn, sstatefixmedir + "/fixmepath")
430 return
431
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500432 bb.note("Replacing fixme paths in sstate package: %s" % (sstate_hardcode_cmd))
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500433 subprocess.check_call(sstate_hardcode_cmd, shell=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500434
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800435 # Need to remove this or we'd copy it into the target directory and may
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500436 # conflict with another writer
437 os.remove(fixmefn)
438}
439
440def sstate_clean_cachefile(ss, d):
441 import oe.path
442
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500443 sstatepkgfile = d.getVar('SSTATE_PATHSPEC') + "*_" + ss['task'] + ".tgz*"
Brad Bishopa5c52ff2018-11-23 10:55:50 +1300444 if d.getVarFlag('do_%s' % ss['task'], 'task'):
445 bb.note("Removing %s" % sstatepkgfile)
446 oe.path.remove(sstatepkgfile)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500447
448def sstate_clean_cachefiles(d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500449 for task in (d.getVar('SSTATETASKS') or "").split():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500450 ld = d.createCopy()
451 ss = sstate_state_fromvars(ld, task)
452 sstate_clean_cachefile(ss, ld)
453
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500454def sstate_clean_manifest(manifest, d, prefix=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500455 import oe.path
456
457 mfile = open(manifest)
458 entries = mfile.readlines()
459 mfile.close()
460
461 for entry in entries:
462 entry = entry.strip()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500463 if prefix and not entry.startswith("/"):
464 entry = prefix + "/" + entry
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500465 bb.debug(2, "Removing manifest: %s" % entry)
466 # We can race against another package populating directories as we're removing them
467 # so we ignore errors here.
468 try:
469 if entry.endswith("/"):
470 if os.path.islink(entry[:-1]):
471 os.remove(entry[:-1])
472 elif os.path.exists(entry) and len(os.listdir(entry)) == 0:
473 os.rmdir(entry[:-1])
474 else:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500475 os.remove(entry)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500476 except OSError:
477 pass
478
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600479 postrm = manifest + ".postrm"
480 if os.path.exists(manifest + ".postrm"):
481 import subprocess
482 os.chmod(postrm, 0o755)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500483 subprocess.check_call(postrm, shell=True)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600484 oe.path.remove(postrm)
485
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500486 oe.path.remove(manifest)
487
488def sstate_clean(ss, d):
489 import oe.path
490 import glob
491
492 d2 = d.createCopy()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500493 stamp_clean = d.getVar("STAMPCLEAN")
494 extrainf = d.getVarFlag("do_" + ss['task'], 'stamp-extra-info')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500495 if extrainf:
496 d2.setVar("SSTATE_MANMACH", extrainf)
497 wildcard_stfile = "%s.do_%s*.%s" % (stamp_clean, ss['task'], extrainf)
498 else:
499 wildcard_stfile = "%s.do_%s*" % (stamp_clean, ss['task'])
500
501 manifest = d2.expand("${SSTATE_MANFILEPREFIX}.%s" % ss['task'])
502
503 if os.path.exists(manifest):
504 locks = []
505 for lock in ss['lockfiles-shared']:
506 locks.append(bb.utils.lockfile(lock))
507 for lock in ss['lockfiles']:
508 locks.append(bb.utils.lockfile(lock))
509
510 sstate_clean_manifest(manifest, d)
511
512 for lock in locks:
513 bb.utils.unlockfile(lock)
514
515 # Remove the current and previous stamps, but keep the sigdata.
516 #
517 # The glob() matches do_task* which may match multiple tasks, for
518 # example: do_package and do_package_write_ipk, so we need to
519 # exactly match *.do_task.* and *.do_task_setscene.*
520 rm_stamp = '.do_%s.' % ss['task']
521 rm_setscene = '.do_%s_setscene.' % ss['task']
522 # For BB_SIGNATURE_HANDLER = "noop"
523 rm_nohash = ".do_%s" % ss['task']
524 for stfile in glob.glob(wildcard_stfile):
525 # Keep the sigdata
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500526 if ".sigdata." in stfile or ".sigbasedata." in stfile:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500527 continue
528 # Preserve taint files in the stamps directory
529 if stfile.endswith('.taint'):
530 continue
531 if rm_stamp in stfile or rm_setscene in stfile or \
532 stfile.endswith(rm_nohash):
533 oe.path.remove(stfile)
534
535sstate_clean[vardepsexclude] = "SSTATE_MANFILEPREFIX"
536
537CLEANFUNCS += "sstate_cleanall"
538
539python sstate_cleanall() {
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500540 bb.note("Removing shared state for package %s" % d.getVar('PN'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500541
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500542 manifest_dir = d.getVar('SSTATE_MANIFESTS')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500543 if not os.path.exists(manifest_dir):
544 return
545
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500546 tasks = d.getVar('SSTATETASKS').split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500547 for name in tasks:
548 ld = d.createCopy()
549 shared_state = sstate_state_fromvars(ld, name)
550 sstate_clean(shared_state, ld)
551}
552
553python sstate_hardcode_path () {
554 import subprocess, platform
555
556 # Need to remove hardcoded paths and fix these when we install the
557 # staging packages.
558 #
559 # Note: the logic in this function needs to match the reverse logic
560 # in sstate_installpkg(ss, d)
561
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500562 staging_target = d.getVar('RECIPE_SYSROOT')
563 staging_host = d.getVar('RECIPE_SYSROOT_NATIVE')
564 sstate_builddir = d.getVar('SSTATE_BUILDDIR')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500565
Brad Bishop316dfdd2018-06-25 12:45:53 -0400566 sstate_sed_cmd = "sed -i -e 's:%s:FIXMESTAGINGDIRHOST:g'" % staging_host
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500567 if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross-canadian', d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500568 sstate_grep_cmd = "grep -l -e '%s'" % (staging_host)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500569 elif bb.data.inherits_class('cross', d) or bb.data.inherits_class('crosssdk', d):
570 sstate_grep_cmd = "grep -l -e '%s' -e '%s'" % (staging_target, staging_host)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400571 sstate_sed_cmd += " -e 's:%s:FIXMESTAGINGDIRTARGET:g'" % staging_target
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500572 else:
Brad Bishop316dfdd2018-06-25 12:45:53 -0400573 sstate_grep_cmd = "grep -l -e '%s' -e '%s'" % (staging_target, staging_host)
574 sstate_sed_cmd += " -e 's:%s:FIXMESTAGINGDIRTARGET:g'" % staging_target
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500575
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500576 extra_staging_fixmes = d.getVar('EXTRA_STAGING_FIXMES') or ''
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500577 for fixmevar in extra_staging_fixmes.split():
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500578 fixme_path = d.getVar(fixmevar)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500579 sstate_sed_cmd += " -e 's:%s:FIXME_%s:g'" % (fixme_path, fixmevar)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500580 sstate_grep_cmd += " -e '%s'" % (fixme_path)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500581
582 fixmefn = sstate_builddir + "fixmepath"
583
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500584 sstate_scan_cmd = d.getVar('SSTATE_SCAN_CMD')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500585 sstate_filelist_cmd = "tee %s" % (fixmefn)
586
587 # fixmepath file needs relative paths, drop sstate_builddir prefix
588 sstate_filelist_relative_cmd = "sed -i -e 's:^%s::g' %s" % (sstate_builddir, fixmefn)
589
590 xargs_no_empty_run_cmd = '--no-run-if-empty'
591 if platform.system() == 'Darwin':
592 xargs_no_empty_run_cmd = ''
593
594 # Limit the fixpaths and sed operations based on the initial grep search
595 # This has the side effect of making sure the vfs cache is hot
596 sstate_hardcode_cmd = "%s | xargs %s | %s | xargs %s %s" % (sstate_scan_cmd, sstate_grep_cmd, sstate_filelist_cmd, xargs_no_empty_run_cmd, sstate_sed_cmd)
597
598 bb.note("Removing hardcoded paths from sstate package: '%s'" % (sstate_hardcode_cmd))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500599 subprocess.check_output(sstate_hardcode_cmd, shell=True, cwd=sstate_builddir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500600
601 # If the fixmefn is empty, remove it..
602 if os.stat(fixmefn).st_size == 0:
603 os.remove(fixmefn)
604 else:
605 bb.note("Replacing absolute paths in fixmepath file: '%s'" % (sstate_filelist_relative_cmd))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500606 subprocess.check_output(sstate_filelist_relative_cmd, shell=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500607}
608
609def sstate_package(ss, d):
610 import oe.path
611
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500612 tmpdir = d.getVar('TMPDIR')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500613
614 sstatebuild = d.expand("${WORKDIR}/sstate-build-%s/" % ss['task'])
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500615 sstatepkg = d.getVar('SSTATE_PKG') + '_'+ ss['task'] + ".tgz"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500616 bb.utils.remove(sstatebuild, recurse=True)
617 bb.utils.mkdirhier(sstatebuild)
618 bb.utils.mkdirhier(os.path.dirname(sstatepkg))
619 for state in ss['dirs']:
620 if not os.path.exists(state[1]):
621 continue
622 srcbase = state[0].rstrip("/").rsplit('/', 1)[0]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500623 # Find and error for absolute symlinks. We could attempt to relocate but its not
624 # clear where the symlink is relative to in this context. We could add that markup
625 # to sstate tasks but there aren't many of these so better just avoid them entirely.
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500626 for walkroot, dirs, files in os.walk(state[1]):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500627 for file in files + dirs:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500628 srcpath = os.path.join(walkroot, file)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500629 if not os.path.islink(srcpath):
630 continue
631 link = os.readlink(srcpath)
632 if not os.path.isabs(link):
633 continue
634 if not link.startswith(tmpdir):
635 continue
636 bb.error("sstate found an absolute path symlink %s pointing at %s. Please replace this with a relative link." % (srcpath, link))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500637 bb.debug(2, "Preparing tree %s for packaging at %s" % (state[1], sstatebuild + state[0]))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500638 os.rename(state[1], sstatebuild + state[0])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500639
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500640 workdir = d.getVar('WORKDIR')
Brad Bishop977dc1a2019-02-06 16:01:43 -0500641 sharedworkdir = os.path.join(d.getVar('TMPDIR'), "work-shared")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500642 for plain in ss['plaindirs']:
643 pdir = plain.replace(workdir, sstatebuild)
Brad Bishop977dc1a2019-02-06 16:01:43 -0500644 if sharedworkdir in plain:
645 pdir = plain.replace(sharedworkdir, sstatebuild)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500646 bb.utils.mkdirhier(plain)
647 bb.utils.mkdirhier(pdir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500648 os.rename(plain, pdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500649
650 d.setVar('SSTATE_BUILDDIR', sstatebuild)
651 d.setVar('SSTATE_PKG', sstatepkg)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500652 d.setVar('SSTATE_INSTDIR', sstatebuild)
653
654 if d.getVar('SSTATE_SKIP_CREATION') == '1':
655 return
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500656
Brad Bishop08902b02019-08-20 09:16:51 -0400657 sstate_create_package = ['sstate_report_unihash', 'sstate_create_package']
658 if d.getVar('SSTATE_SIG_KEY'):
659 sstate_create_package.append('sstate_sign_package')
660
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500661 for f in (d.getVar('SSTATECREATEFUNCS') or '').split() + \
Brad Bishop08902b02019-08-20 09:16:51 -0400662 sstate_create_package + \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500663 (d.getVar('SSTATEPOSTCREATEFUNCS') or '').split():
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500664 # All hooks should run in SSTATE_BUILDDIR.
665 bb.build.exec_func(f, d, (sstatebuild,))
666
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500667 bb.siggen.dump_this_task(sstatepkg + ".siginfo", d)
668
669 return
670
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800671def pstaging_fetch(sstatefetch, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500672 import bb.fetch2
673
674 # Only try and fetch if the user has configured a mirror
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500675 mirrors = d.getVar('SSTATE_MIRRORS')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500676 if not mirrors:
677 return
678
679 # Copy the data object and override DL_DIR and SRC_URI
680 localdata = bb.data.createCopy(d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500681
682 dldir = localdata.expand("${SSTATE_DIR}")
683 bb.utils.mkdirhier(dldir)
684
685 localdata.delVar('MIRRORS')
686 localdata.setVar('FILESPATH', dldir)
687 localdata.setVar('DL_DIR', dldir)
688 localdata.setVar('PREMIRRORS', mirrors)
689
690 # if BB_NO_NETWORK is set but we also have SSTATE_MIRROR_ALLOW_NETWORK,
691 # we'll want to allow network access for the current set of fetches.
Brad Bishopd89cb5f2019-04-10 09:02:41 -0400692 if bb.utils.to_boolean(localdata.getVar('BB_NO_NETWORK')) and \
693 bb.utils.to_boolean(localdata.getVar('SSTATE_MIRROR_ALLOW_NETWORK')):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500694 localdata.delVar('BB_NO_NETWORK')
695
696 # Try a fetch from the sstate mirror, if it fails just return and
697 # we will build the package
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600698 uris = ['file://{0};downloadfilename={0}'.format(sstatefetch),
699 'file://{0}.siginfo;downloadfilename={0}.siginfo'.format(sstatefetch)]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500700 if bb.utils.to_boolean(d.getVar("SSTATE_VERIFY_SIG"), False):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600701 uris += ['file://{0}.sig;downloadfilename={0}.sig'.format(sstatefetch)]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500702
703 for srcuri in uris:
704 localdata.setVar('SRC_URI', srcuri)
705 try:
706 fetcher = bb.fetch2.Fetch([srcuri], localdata, cache=False)
707 fetcher.download()
708
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500709 except bb.fetch2.BBFetchException:
710 break
711
712def sstate_setscene(d):
713 shared_state = sstate_state_fromvars(d)
714 accelerate = sstate_installpkg(shared_state, d)
715 if not accelerate:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600716 bb.fatal("No suitable staging package found")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500717
718python sstate_task_prefunc () {
719 shared_state = sstate_state_fromvars(d)
720 sstate_clean(shared_state, d)
721}
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500722sstate_task_prefunc[dirs] = "${WORKDIR}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500723
724python sstate_task_postfunc () {
725 shared_state = sstate_state_fromvars(d)
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500726
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500727 for intercept in shared_state['interceptfuncs']:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500728 bb.build.exec_func(intercept, d, (d.getVar("WORKDIR"),))
729
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600730 omask = os.umask(0o002)
731 if omask != 0o002:
732 bb.note("Using umask 0o002 (not %0o) for sstate packaging" % omask)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500733 sstate_package(shared_state, d)
734 os.umask(omask)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500735
736 sstateinst = d.getVar("SSTATE_INSTDIR")
737 d.setVar('SSTATE_FIXMEDIR', shared_state['fixmedir'])
738
739 sstate_installpkgdir(shared_state, d)
740
741 bb.utils.remove(d.getVar("SSTATE_BUILDDIR"), recurse=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500742}
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500743sstate_task_postfunc[dirs] = "${WORKDIR}"
744
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500745
746#
747# Shell function to generate a sstate package from a directory
748# set as SSTATE_BUILDDIR. Will be run from within SSTATE_BUILDDIR.
749#
750sstate_create_package () {
751 TFILE=`mktemp ${SSTATE_PKG}.XXXXXXXX`
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800752
Brad Bishop08902b02019-08-20 09:16:51 -0400753 # Exit earlu if it already exists
754 if [ -e ${SSTATE_PKG} ]; then
755 return
756 fi
757
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800758 # Use pigz if available
759 OPT="-czS"
760 if [ -x "$(command -v pigz)" ]; then
761 OPT="-I pigz -cS"
762 fi
763
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500764 # Need to handle empty directories
765 if [ "$(ls -A)" ]; then
766 set +e
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800767 tar $OPT -f $TFILE *
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500768 ret=$?
769 if [ $ret -ne 0 ] && [ $ret -ne 1 ]; then
770 exit 1
771 fi
772 set -e
773 else
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800774 tar $OPT --file=$TFILE --files-from=/dev/null
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500775 fi
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500776 chmod 0664 $TFILE
Brad Bishop08902b02019-08-20 09:16:51 -0400777 # Skip if it was already created by some other process
778 if [ ! -e ${SSTATE_PKG} ]; then
779 mv -f $TFILE ${SSTATE_PKG}
780 else
781 rm $TFILE
782 fi
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500783}
784
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500785python sstate_sign_package () {
786 from oe.gpg_sign import get_signer
787
Brad Bishop08902b02019-08-20 09:16:51 -0400788
789 signer = get_signer(d, 'local')
790 sstate_pkg = d.getVar('SSTATE_PKG')
791 if os.path.exists(sstate_pkg + '.sig'):
792 os.unlink(sstate_pkg + '.sig')
793 signer.detach_sign(sstate_pkg, d.getVar('SSTATE_SIG_KEY', False), None,
794 d.getVar('SSTATE_SIG_PASSPHRASE'), armor=False)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500795}
796
Brad Bishop19323692019-04-05 15:28:33 -0400797python sstate_report_unihash() {
798 report_unihash = getattr(bb.parse.siggen, 'report_unihash', None)
799
800 if report_unihash:
801 ss = sstate_state_fromvars(d)
802 report_unihash(os.getcwd(), ss['task'], d)
803}
804
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500805#
806# Shell function to decompress and prepare a package for installation
807# Will be run from within SSTATE_INSTDIR.
808#
809sstate_unpack_package () {
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500810 tar -xvzf ${SSTATE_PKG}
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500811 # update .siginfo atime on local/NFS mirror
812 [ -w ${SSTATE_PKG}.siginfo ] && [ -h ${SSTATE_PKG}.siginfo ] && touch -a ${SSTATE_PKG}.siginfo
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500813 # Use "! -w ||" to return true for read only files
814 [ ! -w ${SSTATE_PKG} ] || touch --no-dereference ${SSTATE_PKG}
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500815 [ ! -w ${SSTATE_PKG}.sig ] || [ ! -e ${SSTATE_PKG}.sig ] || touch --no-dereference ${SSTATE_PKG}.sig
816 [ ! -w ${SSTATE_PKG}.siginfo ] || [ ! -e ${SSTATE_PKG}.siginfo ] || touch --no-dereference ${SSTATE_PKG}.siginfo
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500817}
818
819BB_HASHCHECK_FUNCTION = "sstate_checkhashes"
820
Brad Bishop08902b02019-08-20 09:16:51 -0400821def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, **kwargs):
822 found = set()
823 missed = set()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500824 extension = ".tgz"
825 if siginfo:
826 extension = extension + ".siginfo"
827
Brad Bishop19323692019-04-05 15:28:33 -0400828 def gethash(task):
Brad Bishop08902b02019-08-20 09:16:51 -0400829 return sq_data['unihash'][task]
Brad Bishop19323692019-04-05 15:28:33 -0400830
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500831 def getpathcomponents(task, d):
832 # Magic data from BB_HASHFILENAME
Brad Bishop08902b02019-08-20 09:16:51 -0400833 splithashfn = sq_data['hashfn'][task].split(" ")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500834 spec = splithashfn[1]
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500835 if splithashfn[0] == "True":
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500836 extrapath = d.getVar("NATIVELSBSTRING") + "/"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500837 else:
838 extrapath = ""
Brad Bishop08902b02019-08-20 09:16:51 -0400839
840 tname = bb.runqueue.taskname_from_tid(task)[3:]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500841
842 if tname in ["fetch", "unpack", "patch", "populate_lic", "preconfigure"] and splithashfn[2]:
843 spec = splithashfn[2]
844 extrapath = ""
845
846 return spec, extrapath, tname
847
848
Brad Bishop08902b02019-08-20 09:16:51 -0400849 for tid in sq_data['hash']:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500850
Brad Bishop08902b02019-08-20 09:16:51 -0400851 spec, extrapath, tname = getpathcomponents(tid, d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500852
Brad Bishop08902b02019-08-20 09:16:51 -0400853 sstatefile = d.expand("${SSTATE_DIR}/" + extrapath + generate_sstatefn(spec, gethash(tid), d) + "_" + tname + extension)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500854
855 if os.path.exists(sstatefile):
856 bb.debug(2, "SState: Found valid sstate file %s" % sstatefile)
Brad Bishop08902b02019-08-20 09:16:51 -0400857 found.add(tid)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500858 continue
859 else:
Brad Bishop08902b02019-08-20 09:16:51 -0400860 missed.add(tid)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500861 bb.debug(2, "SState: Looked for but didn't find file %s" % sstatefile)
862
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500863 mirrors = d.getVar("SSTATE_MIRRORS")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500864 if mirrors:
865 # Copy the data object and override DL_DIR and SRC_URI
866 localdata = bb.data.createCopy(d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500867
868 dldir = localdata.expand("${SSTATE_DIR}")
869 localdata.delVar('MIRRORS')
870 localdata.setVar('FILESPATH', dldir)
871 localdata.setVar('DL_DIR', dldir)
872 localdata.setVar('PREMIRRORS', mirrors)
873
874 bb.debug(2, "SState using premirror of: %s" % mirrors)
875
876 # if BB_NO_NETWORK is set but we also have SSTATE_MIRROR_ALLOW_NETWORK,
877 # we'll want to allow network access for the current set of fetches.
Brad Bishopd89cb5f2019-04-10 09:02:41 -0400878 if bb.utils.to_boolean(localdata.getVar('BB_NO_NETWORK')) and \
879 bb.utils.to_boolean(localdata.getVar('SSTATE_MIRROR_ALLOW_NETWORK')):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500880 localdata.delVar('BB_NO_NETWORK')
881
882 from bb.fetch2 import FetchConnectionCache
883 def checkstatus_init(thread_worker):
884 thread_worker.connection_cache = FetchConnectionCache()
885
886 def checkstatus_end(thread_worker):
887 thread_worker.connection_cache.close_connections()
888
889 def checkstatus(thread_worker, arg):
Brad Bishop08902b02019-08-20 09:16:51 -0400890 (tid, sstatefile) = arg
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500891
892 localdata2 = bb.data.createCopy(localdata)
893 srcuri = "file://" + sstatefile
894 localdata.setVar('SRC_URI', srcuri)
895 bb.debug(2, "SState: Attempting to fetch %s" % srcuri)
896
897 try:
898 fetcher = bb.fetch2.Fetch(srcuri.split(), localdata2,
899 connection_cache=thread_worker.connection_cache)
900 fetcher.checkstatus()
901 bb.debug(2, "SState: Successful fetch test for %s" % srcuri)
Brad Bishop08902b02019-08-20 09:16:51 -0400902 found.add(tid)
903 if tid in missed:
904 missed.remove(tid)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500905 except:
Brad Bishop08902b02019-08-20 09:16:51 -0400906 missed.add(tid)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500907 bb.debug(2, "SState: Unsuccessful fetch test for %s" % srcuri)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600908 pass
Brad Bishop316dfdd2018-06-25 12:45:53 -0400909 bb.event.fire(bb.event.ProcessProgress(msg, len(tasklist) - thread_worker.tasks.qsize()), d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500910
911 tasklist = []
Brad Bishop08902b02019-08-20 09:16:51 -0400912 for tid in sq_data['hash']:
913 if tid in found:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500914 continue
Brad Bishop08902b02019-08-20 09:16:51 -0400915 spec, extrapath, tname = getpathcomponents(tid, d)
916 sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(tid), d) + "_" + tname + extension)
917 tasklist.append((tid, sstatefile))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500918
919 if tasklist:
Brad Bishop316dfdd2018-06-25 12:45:53 -0400920 msg = "Checking sstate mirror object availability"
921 bb.event.fire(bb.event.ProcessStarted(msg, len(tasklist)), d)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600922
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500923 import multiprocessing
924 nproc = min(multiprocessing.cpu_count(), len(tasklist))
925
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600926 bb.event.enable_threadlock()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500927 pool = oe.utils.ThreadedPool(nproc, len(tasklist),
928 worker_init=checkstatus_init, worker_end=checkstatus_end)
929 for t in tasklist:
930 pool.add_task(checkstatus, t)
931 pool.start()
932 pool.wait_completion()
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600933 bb.event.disable_threadlock()
934
Brad Bishop316dfdd2018-06-25 12:45:53 -0400935 bb.event.fire(bb.event.ProcessFinished(msg), d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500936
Brad Bishop96ff1982019-08-19 13:50:42 -0400937 # Likely checking an individual task hash again for multiconfig sharing of sstate tasks so skip reporting
Brad Bishop08902b02019-08-20 09:16:51 -0400938 if len(sq_data['hash']) == 1:
939 return found
Brad Bishop96ff1982019-08-19 13:50:42 -0400940
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500941 inheritlist = d.getVar("INHERIT")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500942 if "toaster" in inheritlist:
943 evdata = {'missed': [], 'found': []};
Brad Bishop08902b02019-08-20 09:16:51 -0400944 for tid in missed:
945 spec, extrapath, tname = getpathcomponents(tid, d)
946 sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(tid), d) + "_" + tname + ".tgz")
947 evdata['missed'].append((bb.runqueue.fn_from_tid(tid), bb.runqueue.taskname_from_tid(tid), gethash(tid), sstatefile ) )
948 for tid in found:
949 spec, extrapath, tname = getpathcomponents(tid, d)
950 sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(tid), d) + "_" + tname + ".tgz")
951 evdata['found'].append((bb.runqueue.fn_from_tid(tid), bb.runqueue.taskname_from_tid(tid), gethash(tid), sstatefile ) )
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500952 bb.event.fire(bb.event.MetadataEvent("MissedSstate", evdata), d)
953
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800954 # Print some summary statistics about the current task completion and how much sstate
955 # reuse there was. Avoid divide by zero errors.
Brad Bishop08902b02019-08-20 09:16:51 -0400956 total = len(sq_data['hash'])
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800957 complete = 0
958 if currentcount:
Brad Bishop08902b02019-08-20 09:16:51 -0400959 complete = (len(found) + currentcount) / (total + currentcount) * 100
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800960 match = 0
961 if total:
Brad Bishop08902b02019-08-20 09:16:51 -0400962 match = len(found) / total * 100
963 bb.plain("Sstate summary: Wanted %d Found %d Missed %d Current %d (%d%% match, %d%% complete)" % (total, len(found), len(missed), currentcount, match, complete))
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800964
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500965 if hasattr(bb.parse.siggen, "checkhashes"):
Brad Bishop08902b02019-08-20 09:16:51 -0400966 bb.parse.siggen.checkhashes(sq_data, missed, found, d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500967
Brad Bishop08902b02019-08-20 09:16:51 -0400968 return found
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500969
970BB_SETSCENE_DEPVALID = "setscene_depvalid"
971
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500972def setscene_depvalid(task, taskdependees, notneeded, d, log=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500973 # taskdependees is a dict of tasks which depend on task, each being a 3 item list of [PN, TASKNAME, FILENAME]
974 # task is included in taskdependees too
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500975 # Return - False - We need this dependency
976 # - True - We can skip this dependency
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800977 import re
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500978
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500979 def logit(msg, log):
980 if log is not None:
981 log.append(msg)
982 else:
983 bb.debug(2, msg)
984
985 logit("Considering setscene task: %s" % (str(taskdependees[task])), log)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500986
987 def isNativeCross(x):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500988 return x.endswith("-native") or "-cross-" in x or "-crosssdk" in x or x.endswith("-cross")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500989
990 # We only need to trigger populate_lic through direct dependencies
991 if taskdependees[task][1] == "do_populate_lic":
992 return True
993
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500994 # stash_locale and gcc_stash_builddir are never needed as a dependency for built objects
995 if taskdependees[task][1] == "do_stash_locale" or taskdependees[task][1] == "do_gcc_stash_builddir":
996 return True
997
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500998 # We only need to trigger packagedata through direct dependencies
999 # but need to preserve packagedata on packagedata links
1000 if taskdependees[task][1] == "do_packagedata":
1001 for dep in taskdependees:
1002 if taskdependees[dep][1] == "do_packagedata":
1003 return False
1004 return True
1005
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001006 for dep in taskdependees:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001007 logit(" considering dependency: %s" % (str(taskdependees[dep])), log)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001008 if task == dep:
1009 continue
1010 if dep in notneeded:
1011 continue
1012 # do_package_write_* and do_package doesn't need do_package
1013 if taskdependees[task][1] == "do_package" and taskdependees[dep][1] in ['do_package', 'do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm', 'do_packagedata', 'do_package_qa']:
1014 continue
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001015 # do_package_write_* need do_populate_sysroot as they're mainly postinstall dependencies
1016 if taskdependees[task][1] == "do_populate_sysroot" and taskdependees[dep][1] in ['do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm']:
1017 return False
1018 # do_package/packagedata/package_qa don't need do_populate_sysroot
1019 if taskdependees[task][1] == "do_populate_sysroot" and taskdependees[dep][1] in ['do_package', 'do_packagedata', 'do_package_qa']:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001020 continue
1021 # Native/Cross packages don't exist and are noexec anyway
1022 if isNativeCross(taskdependees[dep][0]) and taskdependees[dep][1] in ['do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm', 'do_packagedata', 'do_package', 'do_package_qa']:
1023 continue
1024
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001025 # This is due to the [depends] in useradd.bbclass complicating matters
1026 # The logic *is* reversed here due to the way hard setscene dependencies are injected
1027 if (taskdependees[task][1] == 'do_package' or taskdependees[task][1] == 'do_populate_sysroot') and taskdependees[dep][0].endswith(('shadow-native', 'shadow-sysroot', 'base-passwd', 'pseudo-native')) and taskdependees[dep][1] == 'do_populate_sysroot':
1028 continue
1029
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001030 # Consider sysroot depending on sysroot tasks
1031 if taskdependees[task][1] == 'do_populate_sysroot' and taskdependees[dep][1] == 'do_populate_sysroot':
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001032 # Allow excluding certain recursive dependencies. If a recipe needs it should add a
1033 # specific dependency itself, rather than relying on one of its dependees to pull
1034 # them in.
1035 # See also http://lists.openembedded.org/pipermail/openembedded-core/2018-January/146324.html
1036 not_needed = False
1037 excludedeps = d.getVar('_SSTATE_EXCLUDEDEPS_SYSROOT')
1038 if excludedeps is None:
1039 # Cache the regular expressions for speed
1040 excludedeps = []
1041 for excl in (d.getVar('SSTATE_EXCLUDEDEPS_SYSROOT') or "").split():
1042 excludedeps.append((re.compile(excl.split('->', 1)[0]), re.compile(excl.split('->', 1)[1])))
1043 d.setVar('_SSTATE_EXCLUDEDEPS_SYSROOT', excludedeps)
1044 for excl in excludedeps:
1045 if excl[0].match(taskdependees[dep][0]):
1046 if excl[1].match(taskdependees[task][0]):
1047 not_needed = True
1048 break
1049 if not_needed:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001050 continue
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001051 # For meta-extsdk-toolchain we want all sysroot dependencies
1052 if taskdependees[dep][0] == 'meta-extsdk-toolchain':
1053 return False
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001054 # Native/Cross populate_sysroot need their dependencies
1055 if isNativeCross(taskdependees[task][0]) and isNativeCross(taskdependees[dep][0]):
1056 return False
1057 # Target populate_sysroot depended on by cross tools need to be installed
1058 if isNativeCross(taskdependees[dep][0]):
1059 return False
1060 # Native/cross tools depended upon by target sysroot are not needed
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001061 # Add an exception for shadow-native as required by useradd.bbclass
1062 if isNativeCross(taskdependees[task][0]) and taskdependees[task][0] != 'shadow-native':
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001063 continue
1064 # Target populate_sysroot need their dependencies
1065 return False
1066
1067 if taskdependees[task][1] == 'do_shared_workdir':
1068 continue
1069
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001070 if taskdependees[dep][1] == "do_populate_lic":
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001071 continue
1072
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001073
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001074 # Safe fallthrough default
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001075 logit(" Default setscene dependency fall through due to dependency: %s" % (str(taskdependees[dep])), log)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001076 return False
1077 return True
1078
1079addhandler sstate_eventhandler
1080sstate_eventhandler[eventmask] = "bb.build.TaskSucceeded"
1081python sstate_eventhandler() {
1082 d = e.data
1083 # When we write an sstate package we rewrite the SSTATE_PKG
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001084 spkg = d.getVar('SSTATE_PKG')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001085 if not spkg.endswith(".tgz"):
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001086 taskname = d.getVar("BB_RUNTASK")[3:]
1087 spec = d.getVar('SSTATE_PKGSPEC')
1088 swspec = d.getVar('SSTATE_SWSPEC')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001089 if taskname in ["fetch", "unpack", "patch", "populate_lic", "preconfigure"] and swspec:
1090 d.setVar("SSTATE_PKGSPEC", "${SSTATE_SWSPEC}")
1091 d.setVar("SSTATE_EXTRAPATH", "")
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001092 sstatepkg = d.getVar('SSTATE_PKG')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001093 bb.siggen.dump_this_task(sstatepkg + '_' + taskname + ".tgz" ".siginfo", d)
1094}
1095
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001096SSTATE_PRUNE_OBSOLETEWORKDIR ?= "1"
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001097
1098# Event handler which removes manifests and stamps file for
1099# recipes which are no longer reachable in a build where they
1100# once were.
1101# Also optionally removes the workdir of those tasks/recipes
1102#
1103addhandler sstate_eventhandler2
1104sstate_eventhandler2[eventmask] = "bb.event.ReachableStamps"
1105python sstate_eventhandler2() {
1106 import glob
1107 d = e.data
1108 stamps = e.stamps.values()
1109 removeworkdir = (d.getVar("SSTATE_PRUNE_OBSOLETEWORKDIR", False) == "1")
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001110 preservestampfile = d.expand('${SSTATE_MANIFESTS}/preserve-stamps')
1111 preservestamps = []
1112 if os.path.exists(preservestampfile):
1113 with open(preservestampfile, 'r') as f:
1114 preservestamps = f.readlines()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001115 seen = []
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001116
1117 # The machine index contains all the stamps this machine has ever seen in this build directory.
1118 # We should only remove things which this machine once accessed but no longer does.
1119 machineindex = set()
1120 bb.utils.mkdirhier(d.expand("${SSTATE_MANIFESTS}"))
1121 mi = d.expand("${SSTATE_MANIFESTS}/index-machine-${MACHINE}")
1122 if os.path.exists(mi):
1123 with open(mi, "r") as f:
1124 machineindex = set(line.strip() for line in f.readlines())
1125
Brad Bishop316dfdd2018-06-25 12:45:53 -04001126 for a in sorted(list(set(d.getVar("SSTATE_ARCHS").split()))):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001127 toremove = []
1128 i = d.expand("${SSTATE_MANIFESTS}/index-" + a)
1129 if not os.path.exists(i):
1130 continue
1131 with open(i, "r") as f:
1132 lines = f.readlines()
1133 for l in lines:
Brad Bishop19323692019-04-05 15:28:33 -04001134 try:
1135 (stamp, manifest, workdir) = l.split()
1136 if stamp not in stamps and stamp not in preservestamps and stamp in machineindex:
1137 toremove.append(l)
1138 if stamp not in seen:
1139 bb.debug(2, "Stamp %s is not reachable, removing related manifests" % stamp)
1140 seen.append(stamp)
1141 except ValueError:
1142 bb.fatal("Invalid line '%s' in sstate manifest '%s'" % (l, i))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001143
1144 if toremove:
Brad Bishop316dfdd2018-06-25 12:45:53 -04001145 msg = "Removing %d recipes from the %s sysroot" % (len(toremove), a)
1146 bb.event.fire(bb.event.ProcessStarted(msg, len(toremove)), d)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001147
Brad Bishop316dfdd2018-06-25 12:45:53 -04001148 removed = 0
1149 for r in toremove:
1150 (stamp, manifest, workdir) = r.split()
1151 for m in glob.glob(manifest + ".*"):
1152 if m.endswith(".postrm"):
1153 continue
1154 sstate_clean_manifest(m, d)
1155 bb.utils.remove(stamp + "*")
1156 if removeworkdir:
1157 bb.utils.remove(workdir, recurse = True)
1158 lines.remove(r)
1159 removed = removed + 1
1160 bb.event.fire(bb.event.ProcessProgress(msg, removed), d)
1161
1162 bb.event.fire(bb.event.ProcessFinished(msg), d)
1163
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001164 with open(i, "w") as f:
1165 for l in lines:
1166 f.write(l)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001167 machineindex |= set(stamps)
1168 with open(mi, "w") as f:
1169 for l in machineindex:
1170 f.write(l + "\n")
1171
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001172 if preservestamps:
1173 os.remove(preservestampfile)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001174}