blob: 0a12935be9323af26cc3de936f083b4f9b6da937 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001SSTATE_VERSION = "3"
2
3SSTATE_MANIFESTS ?= "${TMPDIR}/sstate-control"
4SSTATE_MANFILEPREFIX = "${SSTATE_MANIFESTS}/manifest-${SSTATE_MANMACH}-${PN}"
5
6def generate_sstatefn(spec, hash, d):
7 if not hash:
8 hash = "INVALID"
9 return hash[:2] + "/" + spec + hash
10
11SSTATE_PKGARCH = "${PACKAGE_ARCH}"
12SSTATE_PKGSPEC = "sstate:${PN}:${PACKAGE_ARCH}${TARGET_VENDOR}-${TARGET_OS}:${PV}:${PR}:${SSTATE_PKGARCH}:${SSTATE_VERSION}:"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050013SSTATE_SWSPEC = "sstate:${PN}::${PV}:${PR}::${SSTATE_VERSION}:"
Brad Bishop6e60e8b2018-02-01 10:27:11 -050014SSTATE_PKGNAME = "${SSTATE_EXTRAPATH}${@generate_sstatefn(d.getVar('SSTATE_PKGSPEC'), d.getVar('BB_TASKHASH'), d)}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050015SSTATE_PKG = "${SSTATE_DIR}/${SSTATE_PKGNAME}"
16SSTATE_EXTRAPATH = ""
17SSTATE_EXTRAPATHWILDCARD = ""
18SSTATE_PATHSPEC = "${SSTATE_DIR}/${SSTATE_EXTRAPATHWILDCARD}*/${SSTATE_PKGSPEC}"
19
Patrick Williamsc0f7c042017-02-23 20:41:17 -060020# explicitly make PV to depend on evaluated value of PV variable
21PV[vardepvalue] = "${PV}"
22
Patrick Williamsc124f4f2015-09-15 14:41:29 -050023# We don't want the sstate to depend on things like the distro string
24# of the system, we let the sstate paths take care of this.
25SSTATE_EXTRAPATH[vardepvalue] = ""
26
27# For multilib rpm the allarch packagegroup files can overwrite (in theory they're identical)
Brad Bishop6e60e8b2018-02-01 10:27:11 -050028SSTATE_DUPWHITELIST = "${DEPLOY_DIR_IMAGE}/ ${DEPLOY_DIR}/licenses/ ${DEPLOY_DIR_RPM}/noarch/"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050029# Avoid docbook/sgml catalog warnings for now
30SSTATE_DUPWHITELIST += "${STAGING_ETCDIR_NATIVE}/sgml ${STAGING_DATADIR_NATIVE}/sgml"
31# Archive the sources for many architectures in one deploy folder
32SSTATE_DUPWHITELIST += "${DEPLOY_DIR_SRC}"
33
Brad Bishop6e60e8b2018-02-01 10:27:11 -050034SSTATE_SCAN_FILES ?= "*.la *-config *_config postinst-*"
35SSTATE_SCAN_CMD ??= 'find ${SSTATE_BUILDDIR} \( -name "${@"\" -o -name \"".join(d.getVar("SSTATE_SCAN_FILES").split())}" \) -type f'
36SSTATE_SCAN_CMD_NATIVE ??= 'grep -Irl -e ${RECIPE_SYSROOT} -e ${RECIPE_SYSROOT_NATIVE} -e ${HOSTTOOLS_DIR} ${SSTATE_BUILDDIR}'
Patrick Williamsc124f4f2015-09-15 14:41:29 -050037
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050038BB_HASHFILENAME = "False ${SSTATE_PKGSPEC} ${SSTATE_SWSPEC}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050039
40SSTATE_ARCHS = " \
41 ${BUILD_ARCH} \
42 ${BUILD_ARCH}_${SDK_ARCH}_${SDK_OS} \
43 ${BUILD_ARCH}_${TARGET_ARCH} \
44 ${SDK_ARCH}_${SDK_OS} \
45 ${SDK_ARCH}_${PACKAGE_ARCH} \
46 allarch \
47 ${PACKAGE_ARCH} \
48 ${MACHINE}"
49
50SSTATE_MANMACH ?= "${SSTATE_PKGARCH}"
51
52SSTATECREATEFUNCS = "sstate_hardcode_path"
53SSTATEPOSTCREATEFUNCS = ""
54SSTATEPREINSTFUNCS = ""
55SSTATEPOSTUNPACKFUNCS = "sstate_hardcode_path_unpack"
56SSTATEPOSTINSTFUNCS = ""
Brad Bishop6e60e8b2018-02-01 10:27:11 -050057EXTRA_STAGING_FIXMES ?= "HOSTTOOLS_DIR"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050058SSTATECLEANFUNCS = ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -050059
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050060# Check whether sstate exists for tasks that support sstate and are in the
61# locked signatures file.
62SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK ?= 'error'
63
64# Check whether the task's computed hash matches the task's hash in the
65# locked signatures file.
66SIGGEN_LOCKEDSIGS_TASKSIG_CHECK ?= "error"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050067
68# The GnuPG key ID and passphrase to use to sign sstate archives (or unset to
69# not sign)
70SSTATE_SIG_KEY ?= ""
71SSTATE_SIG_PASSPHRASE ?= ""
72# Whether to verify the GnUPG signatures when extracting sstate archives
73SSTATE_VERIFY_SIG ?= "0"
74
Patrick Williamsc124f4f2015-09-15 14:41:29 -050075python () {
76 if bb.data.inherits_class('native', d):
77 d.setVar('SSTATE_PKGARCH', d.getVar('BUILD_ARCH', False))
78 elif bb.data.inherits_class('crosssdk', d):
79 d.setVar('SSTATE_PKGARCH', d.expand("${BUILD_ARCH}_${SDK_ARCH}_${SDK_OS}"))
80 elif bb.data.inherits_class('cross', d):
81 d.setVar('SSTATE_PKGARCH', d.expand("${BUILD_ARCH}_${TARGET_ARCH}"))
82 elif bb.data.inherits_class('nativesdk', d):
83 d.setVar('SSTATE_PKGARCH', d.expand("${SDK_ARCH}_${SDK_OS}"))
84 elif bb.data.inherits_class('cross-canadian', d):
85 d.setVar('SSTATE_PKGARCH', d.expand("${SDK_ARCH}_${PACKAGE_ARCH}"))
Brad Bishop6e60e8b2018-02-01 10:27:11 -050086 elif bb.data.inherits_class('allarch', d) and d.getVar("PACKAGE_ARCH") == "all":
Patrick Williamsc124f4f2015-09-15 14:41:29 -050087 d.setVar('SSTATE_PKGARCH', "allarch")
88 else:
89 d.setVar('SSTATE_MANMACH', d.expand("${PACKAGE_ARCH}"))
90
91 if bb.data.inherits_class('native', d) or bb.data.inherits_class('crosssdk', d) or bb.data.inherits_class('cross', d):
92 d.setVar('SSTATE_EXTRAPATH', "${NATIVELSBSTRING}/")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050093 d.setVar('BB_HASHFILENAME', "True ${SSTATE_PKGSPEC} ${SSTATE_SWSPEC}")
Patrick Williamsc124f4f2015-09-15 14:41:29 -050094 d.setVar('SSTATE_EXTRAPATHWILDCARD', "*/")
95
Brad Bishop6e60e8b2018-02-01 10:27:11 -050096 unique_tasks = sorted(set((d.getVar('SSTATETASKS') or "").split()))
Patrick Williamsc124f4f2015-09-15 14:41:29 -050097 d.setVar('SSTATETASKS', " ".join(unique_tasks))
98 for task in unique_tasks:
99 d.prependVarFlag(task, 'prefuncs', "sstate_task_prefunc ")
100 d.appendVarFlag(task, 'postfuncs', " sstate_task_postfunc")
101}
102
103def sstate_init(task, d):
104 ss = {}
105 ss['task'] = task
106 ss['dirs'] = []
107 ss['plaindirs'] = []
108 ss['lockfiles'] = []
109 ss['lockfiles-shared'] = []
110 return ss
111
112def sstate_state_fromvars(d, task = None):
113 if task is None:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500114 task = d.getVar('BB_CURRENTTASK')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500115 if not task:
116 bb.fatal("sstate code running without task context?!")
117 task = task.replace("_setscene", "")
118
119 if task.startswith("do_"):
120 task = task[3:]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500121 inputs = (d.getVarFlag("do_" + task, 'sstate-inputdirs') or "").split()
122 outputs = (d.getVarFlag("do_" + task, 'sstate-outputdirs') or "").split()
123 plaindirs = (d.getVarFlag("do_" + task, 'sstate-plaindirs') or "").split()
124 lockfiles = (d.getVarFlag("do_" + task, 'sstate-lockfile') or "").split()
125 lockfilesshared = (d.getVarFlag("do_" + task, 'sstate-lockfile-shared') or "").split()
126 interceptfuncs = (d.getVarFlag("do_" + task, 'sstate-interceptfuncs') or "").split()
127 fixmedir = d.getVarFlag("do_" + task, 'sstate-fixmedir') or ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500128 if not task or len(inputs) != len(outputs):
129 bb.fatal("sstate variables not setup correctly?!")
130
131 if task == "populate_lic":
132 d.setVar("SSTATE_PKGSPEC", "${SSTATE_SWSPEC}")
133 d.setVar("SSTATE_EXTRAPATH", "")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500134 d.setVar('SSTATE_EXTRAPATHWILDCARD', "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500135
136 ss = sstate_init(task, d)
137 for i in range(len(inputs)):
138 sstate_add(ss, inputs[i], outputs[i], d)
139 ss['lockfiles'] = lockfiles
140 ss['lockfiles-shared'] = lockfilesshared
141 ss['plaindirs'] = plaindirs
142 ss['interceptfuncs'] = interceptfuncs
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500143 ss['fixmedir'] = fixmedir
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500144 return ss
145
146def sstate_add(ss, source, dest, d):
147 if not source.endswith("/"):
148 source = source + "/"
149 if not dest.endswith("/"):
150 dest = dest + "/"
151 source = os.path.normpath(source)
152 dest = os.path.normpath(dest)
153 srcbase = os.path.basename(source)
154 ss['dirs'].append([srcbase, source, dest])
155 return ss
156
157def sstate_install(ss, d):
158 import oe.path
159 import oe.sstatesig
160 import subprocess
161
162 sharedfiles = []
163 shareddirs = []
164 bb.utils.mkdirhier(d.expand("${SSTATE_MANIFESTS}"))
165
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500166 sstateinst = d.expand("${WORKDIR}/sstate-install-%s/" % ss['task'])
167
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500168 manifest, d2 = oe.sstatesig.sstate_get_manifest_filename(ss['task'], d)
169
170 if os.access(manifest, os.R_OK):
171 bb.fatal("Package already staged (%s)?!" % manifest)
172
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600173 d.setVar("SSTATE_INST_POSTRM", manifest + ".postrm")
174
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500175 locks = []
176 for lock in ss['lockfiles-shared']:
177 locks.append(bb.utils.lockfile(lock, True))
178 for lock in ss['lockfiles']:
179 locks.append(bb.utils.lockfile(lock))
180
181 for state in ss['dirs']:
182 bb.debug(2, "Staging files from %s to %s" % (state[1], state[2]))
183 for walkroot, dirs, files in os.walk(state[1]):
184 for file in files:
185 srcpath = os.path.join(walkroot, file)
186 dstpath = srcpath.replace(state[1], state[2])
187 #bb.debug(2, "Staging %s to %s" % (srcpath, dstpath))
188 sharedfiles.append(dstpath)
189 for dir in dirs:
190 srcdir = os.path.join(walkroot, dir)
191 dstdir = srcdir.replace(state[1], state[2])
192 #bb.debug(2, "Staging %s to %s" % (srcdir, dstdir))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500193 if os.path.islink(srcdir):
194 sharedfiles.append(dstdir)
195 continue
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500196 if not dstdir.endswith("/"):
197 dstdir = dstdir + "/"
198 shareddirs.append(dstdir)
199
200 # Check the file list for conflicts against files which already exist
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500201 whitelist = (d.getVar("SSTATE_DUPWHITELIST") or "").split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500202 match = []
203 for f in sharedfiles:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500204 if os.path.exists(f) and not os.path.islink(f):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500205 f = os.path.normpath(f)
206 realmatch = True
207 for w in whitelist:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600208 w = os.path.normpath(w)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500209 if f.startswith(w):
210 realmatch = False
211 break
212 if realmatch:
213 match.append(f)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500214 sstate_search_cmd = "grep -rlF '%s' %s --exclude=master.list | sed -e 's:^.*/::'" % (f, d.expand("${SSTATE_MANIFESTS}"))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500215 search_output = subprocess.Popen(sstate_search_cmd, shell=True, stdout=subprocess.PIPE).communicate()[0]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500216 if search_output:
217 match.append(" (matched in %s)" % search_output.decode('utf-8').rstrip())
218 else:
219 match.append(" (not matched to any task)")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500220 if match:
221 bb.error("The recipe %s is trying to install files into a shared " \
222 "area when those files already exist. Those files and their manifest " \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500223 "location are:\n %s\nPlease verify which recipe should provide the " \
224 "above files.\n\nThe build has stopped, as continuing in this scenario WILL " \
225 "break things - if not now, possibly in the future (we've seen builds fail " \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500226 "several months later). If the system knew how to recover from this " \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500227 "automatically it would, however there are several different scenarios " \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500228 "which can result in this and we don't know which one this is. It may be " \
229 "you have switched providers of something like virtual/kernel (e.g. from " \
230 "linux-yocto to linux-yocto-dev), in that case you need to execute the " \
231 "clean task for both recipes and it will resolve this error. It may be " \
232 "you changed DISTRO_FEATURES from systemd to udev or vice versa. Cleaning " \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500233 "those recipes should again resolve this error, however switching " \
234 "DISTRO_FEATURES on an existing build directory is not supported - you " \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500235 "should really clean out tmp and rebuild (reusing sstate should be safe). " \
236 "It could be the overlapping files detected are harmless in which case " \
237 "adding them to SSTATE_DUPWHITELIST may be the correct solution. It could " \
238 "also be your build is including two different conflicting versions of " \
239 "things (e.g. bluez 4 and bluez 5 and the correct solution for that would " \
240 "be to resolve the conflict. If in doubt, please ask on the mailing list, " \
241 "sharing the error and filelist above." % \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500242 (d.getVar('PN'), "\n ".join(match)))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500243 bb.fatal("If the above message is too much, the simpler version is you're advised to wipe out tmp and rebuild (reusing sstate is fine). That will likely fix things in most (but not all) cases.")
244
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500245 if ss['fixmedir'] and os.path.exists(ss['fixmedir'] + "/fixmepath.cmd"):
246 sharedfiles.append(ss['fixmedir'] + "/fixmepath.cmd")
247 sharedfiles.append(ss['fixmedir'] + "/fixmepath")
248
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500249 # Write out the manifest
250 f = open(manifest, "w")
251 for file in sharedfiles:
252 f.write(file + "\n")
253
254 # We want to ensure that directories appear at the end of the manifest
255 # so that when we test to see if they should be deleted any contents
256 # added by the task will have been removed first.
257 dirs = sorted(shareddirs, key=len)
258 # Must remove children first, which will have a longer path than the parent
259 for di in reversed(dirs):
260 f.write(di + "\n")
261 f.close()
262
263 # Append to the list of manifests for this PACKAGE_ARCH
264
265 i = d2.expand("${SSTATE_MANIFESTS}/index-${SSTATE_MANMACH}")
266 l = bb.utils.lockfile(i + ".lock")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500267 filedata = d.getVar("STAMP") + " " + d2.getVar("SSTATE_MANFILEPREFIX") + " " + d.getVar("WORKDIR") + "\n"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500268 manifests = []
269 if os.path.exists(i):
270 with open(i, "r") as f:
271 manifests = f.readlines()
272 if filedata not in manifests:
273 with open(i, "a+") as f:
274 f.write(filedata)
275 bb.utils.unlockfile(l)
276
277 # Run the actual file install
278 for state in ss['dirs']:
279 if os.path.exists(state[1]):
280 oe.path.copyhardlinktree(state[1], state[2])
281
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500282 for postinst in (d.getVar('SSTATEPOSTINSTFUNCS') or '').split():
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500283 # All hooks should run in the SSTATE_INSTDIR
284 bb.build.exec_func(postinst, d, (sstateinst,))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500285
286 for lock in locks:
287 bb.utils.unlockfile(lock)
288
289sstate_install[vardepsexclude] += "SSTATE_DUPWHITELIST STATE_MANMACH SSTATE_MANFILEPREFIX"
290sstate_install[vardeps] += "${SSTATEPOSTINSTFUNCS}"
291
292def sstate_installpkg(ss, d):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500293 from oe.gpg_sign import get_signer
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500294
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500295 sstateinst = d.expand("${WORKDIR}/sstate-install-%s/" % ss['task'])
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500296 sstatefetch = d.getVar('SSTATE_PKGNAME') + '_' + ss['task'] + ".tgz"
297 sstatepkg = d.getVar('SSTATE_PKG') + '_' + ss['task'] + ".tgz"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500298
299 if not os.path.exists(sstatepkg):
300 pstaging_fetch(sstatefetch, sstatepkg, d)
301
302 if not os.path.isfile(sstatepkg):
303 bb.note("Staging package %s does not exist" % sstatepkg)
304 return False
305
306 sstate_clean(ss, d)
307
308 d.setVar('SSTATE_INSTDIR', sstateinst)
309 d.setVar('SSTATE_PKG', sstatepkg)
310
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500311 if bb.utils.to_boolean(d.getVar("SSTATE_VERIFY_SIG"), False):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500312 signer = get_signer(d, 'local')
313 if not signer.verify(sstatepkg + '.sig'):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500314 bb.warn("Cannot verify signature on sstate package %s" % sstatepkg)
315
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500316 # Empty sstateinst directory, ensure its clean
317 if os.path.exists(sstateinst):
318 oe.path.remove(sstateinst)
319 bb.utils.mkdirhier(sstateinst)
320
321 sstateinst = d.getVar("SSTATE_INSTDIR")
322 d.setVar('SSTATE_FIXMEDIR', ss['fixmedir'])
323
324 for f in (d.getVar('SSTATEPREINSTFUNCS') or '').split() + ['sstate_unpack_package']:
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500325 # All hooks should run in the SSTATE_INSTDIR
326 bb.build.exec_func(f, d, (sstateinst,))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500327
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500328 return sstate_installpkgdir(ss, d)
329
330def sstate_installpkgdir(ss, d):
331 import oe.path
332 import subprocess
333
334 sstateinst = d.getVar("SSTATE_INSTDIR")
335 d.setVar('SSTATE_FIXMEDIR', ss['fixmedir'])
336
337 for f in (d.getVar('SSTATEPOSTUNPACKFUNCS') or '').split():
338 # All hooks should run in the SSTATE_INSTDIR
339 bb.build.exec_func(f, d, (sstateinst,))
340
341 def prepdir(dir):
342 # remove dir if it exists, ensure any parent directories do exist
343 if os.path.exists(dir):
344 oe.path.remove(dir)
345 bb.utils.mkdirhier(dir)
346 oe.path.remove(dir)
347
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500348 for state in ss['dirs']:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500349 if d.getVar('SSTATE_SKIP_CREATION') == '1':
350 continue
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500351 prepdir(state[1])
352 os.rename(sstateinst + state[0], state[1])
353 sstate_install(ss, d)
354
355 for plain in ss['plaindirs']:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500356 workdir = d.getVar('WORKDIR')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500357 src = sstateinst + "/" + plain.replace(workdir, '')
358 dest = plain
359 bb.utils.mkdirhier(src)
360 prepdir(dest)
361 os.rename(src, dest)
362
363 return True
364
365python sstate_hardcode_path_unpack () {
366 # Fixup hardcoded paths
367 #
368 # Note: The logic below must match the reverse logic in
369 # sstate_hardcode_path(d)
370 import subprocess
371
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500372 sstateinst = d.getVar('SSTATE_INSTDIR')
373 sstatefixmedir = d.getVar('SSTATE_FIXMEDIR')
374 fixmefn = sstateinst + "fixmepath"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500375 if os.path.isfile(fixmefn):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500376 staging_target = d.getVar('RECIPE_SYSROOT')
377 staging_host = d.getVar('RECIPE_SYSROOT_NATIVE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500378
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500379 if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross-canadian', d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500380 sstate_sed_cmd = "sed -i -e 's:FIXMESTAGINGDIRHOST:%s:g'" % (staging_host)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500381 elif bb.data.inherits_class('cross', d) or bb.data.inherits_class('crosssdk', d):
382 sstate_sed_cmd = "sed -i -e 's:FIXMESTAGINGDIRTARGET:%s:g; s:FIXMESTAGINGDIRHOST:%s:g'" % (staging_target, staging_host)
383 else:
384 sstate_sed_cmd = "sed -i -e 's:FIXMESTAGINGDIRTARGET:%s:g'" % (staging_target)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500385
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500386 extra_staging_fixmes = d.getVar('EXTRA_STAGING_FIXMES') or ''
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500387 for fixmevar in extra_staging_fixmes.split():
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500388 fixme_path = d.getVar(fixmevar)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500389 sstate_sed_cmd += " -e 's:FIXME_%s:%s:g'" % (fixmevar, fixme_path)
390
391 # Add sstateinst to each filename in fixmepath, use xargs to efficiently call sed
392 sstate_hardcode_cmd = "sed -e 's:^:%s:g' %s | xargs %s" % (sstateinst, fixmefn, sstate_sed_cmd)
393
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500394 # Defer do_populate_sysroot relocation command
395 if sstatefixmedir:
396 bb.utils.mkdirhier(sstatefixmedir)
397 with open(sstatefixmedir + "/fixmepath.cmd", "w") as f:
398 sstate_hardcode_cmd = sstate_hardcode_cmd.replace(fixmefn, sstatefixmedir + "/fixmepath")
399 sstate_hardcode_cmd = sstate_hardcode_cmd.replace(sstateinst, "FIXMEFINALSSTATEINST")
400 sstate_hardcode_cmd = sstate_hardcode_cmd.replace(staging_host, "FIXMEFINALSSTATEHOST")
401 sstate_hardcode_cmd = sstate_hardcode_cmd.replace(staging_target, "FIXMEFINALSSTATETARGET")
402 f.write(sstate_hardcode_cmd)
403 bb.utils.copyfile(fixmefn, sstatefixmedir + "/fixmepath")
404 return
405
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500406 bb.note("Replacing fixme paths in sstate package: %s" % (sstate_hardcode_cmd))
407 subprocess.call(sstate_hardcode_cmd, shell=True)
408
409 # Need to remove this or we'd copy it into the target directory and may
410 # conflict with another writer
411 os.remove(fixmefn)
412}
413
414def sstate_clean_cachefile(ss, d):
415 import oe.path
416
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500417 sstatepkgfile = d.getVar('SSTATE_PATHSPEC') + "*_" + ss['task'] + ".tgz*"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500418 bb.note("Removing %s" % sstatepkgfile)
419 oe.path.remove(sstatepkgfile)
420
421def sstate_clean_cachefiles(d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500422 for task in (d.getVar('SSTATETASKS') or "").split():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500423 ld = d.createCopy()
424 ss = sstate_state_fromvars(ld, task)
425 sstate_clean_cachefile(ss, ld)
426
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500427def sstate_clean_manifest(manifest, d, prefix=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500428 import oe.path
429
430 mfile = open(manifest)
431 entries = mfile.readlines()
432 mfile.close()
433
434 for entry in entries:
435 entry = entry.strip()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500436 if prefix and not entry.startswith("/"):
437 entry = prefix + "/" + entry
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500438 bb.debug(2, "Removing manifest: %s" % entry)
439 # We can race against another package populating directories as we're removing them
440 # so we ignore errors here.
441 try:
442 if entry.endswith("/"):
443 if os.path.islink(entry[:-1]):
444 os.remove(entry[:-1])
445 elif os.path.exists(entry) and len(os.listdir(entry)) == 0:
446 os.rmdir(entry[:-1])
447 else:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500448 os.remove(entry)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500449 except OSError:
450 pass
451
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600452 postrm = manifest + ".postrm"
453 if os.path.exists(manifest + ".postrm"):
454 import subprocess
455 os.chmod(postrm, 0o755)
456 subprocess.call(postrm, shell=True)
457 oe.path.remove(postrm)
458
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500459 oe.path.remove(manifest)
460
461def sstate_clean(ss, d):
462 import oe.path
463 import glob
464
465 d2 = d.createCopy()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500466 stamp_clean = d.getVar("STAMPCLEAN")
467 extrainf = d.getVarFlag("do_" + ss['task'], 'stamp-extra-info')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500468 if extrainf:
469 d2.setVar("SSTATE_MANMACH", extrainf)
470 wildcard_stfile = "%s.do_%s*.%s" % (stamp_clean, ss['task'], extrainf)
471 else:
472 wildcard_stfile = "%s.do_%s*" % (stamp_clean, ss['task'])
473
474 manifest = d2.expand("${SSTATE_MANFILEPREFIX}.%s" % ss['task'])
475
476 if os.path.exists(manifest):
477 locks = []
478 for lock in ss['lockfiles-shared']:
479 locks.append(bb.utils.lockfile(lock))
480 for lock in ss['lockfiles']:
481 locks.append(bb.utils.lockfile(lock))
482
483 sstate_clean_manifest(manifest, d)
484
485 for lock in locks:
486 bb.utils.unlockfile(lock)
487
488 # Remove the current and previous stamps, but keep the sigdata.
489 #
490 # The glob() matches do_task* which may match multiple tasks, for
491 # example: do_package and do_package_write_ipk, so we need to
492 # exactly match *.do_task.* and *.do_task_setscene.*
493 rm_stamp = '.do_%s.' % ss['task']
494 rm_setscene = '.do_%s_setscene.' % ss['task']
495 # For BB_SIGNATURE_HANDLER = "noop"
496 rm_nohash = ".do_%s" % ss['task']
497 for stfile in glob.glob(wildcard_stfile):
498 # Keep the sigdata
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500499 if ".sigdata." in stfile or ".sigbasedata." in stfile:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500500 continue
501 # Preserve taint files in the stamps directory
502 if stfile.endswith('.taint'):
503 continue
504 if rm_stamp in stfile or rm_setscene in stfile or \
505 stfile.endswith(rm_nohash):
506 oe.path.remove(stfile)
507
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500508 # Removes the users/groups created by the package
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500509 for cleanfunc in (d.getVar('SSTATECLEANFUNCS') or '').split():
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500510 bb.build.exec_func(cleanfunc, d)
511
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500512sstate_clean[vardepsexclude] = "SSTATE_MANFILEPREFIX"
513
514CLEANFUNCS += "sstate_cleanall"
515
516python sstate_cleanall() {
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500517 bb.note("Removing shared state for package %s" % d.getVar('PN'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500518
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500519 manifest_dir = d.getVar('SSTATE_MANIFESTS')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500520 if not os.path.exists(manifest_dir):
521 return
522
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500523 tasks = d.getVar('SSTATETASKS').split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500524 for name in tasks:
525 ld = d.createCopy()
526 shared_state = sstate_state_fromvars(ld, name)
527 sstate_clean(shared_state, ld)
528}
529
530python sstate_hardcode_path () {
531 import subprocess, platform
532
533 # Need to remove hardcoded paths and fix these when we install the
534 # staging packages.
535 #
536 # Note: the logic in this function needs to match the reverse logic
537 # in sstate_installpkg(ss, d)
538
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500539 staging_target = d.getVar('RECIPE_SYSROOT')
540 staging_host = d.getVar('RECIPE_SYSROOT_NATIVE')
541 sstate_builddir = d.getVar('SSTATE_BUILDDIR')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500542
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500543 if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross-canadian', d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500544 sstate_grep_cmd = "grep -l -e '%s'" % (staging_host)
545 sstate_sed_cmd = "sed -i -e 's:%s:FIXMESTAGINGDIRHOST:g'" % (staging_host)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500546 elif bb.data.inherits_class('cross', d) or bb.data.inherits_class('crosssdk', d):
547 sstate_grep_cmd = "grep -l -e '%s' -e '%s'" % (staging_target, staging_host)
548 sstate_sed_cmd = "sed -i -e 's:%s:FIXMESTAGINGDIRTARGET:g; s:%s:FIXMESTAGINGDIRHOST:g'" % (staging_target, staging_host)
549 else:
550 sstate_grep_cmd = "grep -l -e '%s'" % (staging_target)
551 sstate_sed_cmd = "sed -i -e 's:%s:FIXMESTAGINGDIRTARGET:g'" % (staging_target)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500552
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500553 extra_staging_fixmes = d.getVar('EXTRA_STAGING_FIXMES') or ''
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500554 for fixmevar in extra_staging_fixmes.split():
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500555 fixme_path = d.getVar(fixmevar)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500556 sstate_sed_cmd += " -e 's:%s:FIXME_%s:g'" % (fixme_path, fixmevar)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500557 sstate_grep_cmd += " -e '%s'" % (fixme_path)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500558
559 fixmefn = sstate_builddir + "fixmepath"
560
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500561 sstate_scan_cmd = d.getVar('SSTATE_SCAN_CMD')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500562 sstate_filelist_cmd = "tee %s" % (fixmefn)
563
564 # fixmepath file needs relative paths, drop sstate_builddir prefix
565 sstate_filelist_relative_cmd = "sed -i -e 's:^%s::g' %s" % (sstate_builddir, fixmefn)
566
567 xargs_no_empty_run_cmd = '--no-run-if-empty'
568 if platform.system() == 'Darwin':
569 xargs_no_empty_run_cmd = ''
570
571 # Limit the fixpaths and sed operations based on the initial grep search
572 # This has the side effect of making sure the vfs cache is hot
573 sstate_hardcode_cmd = "%s | xargs %s | %s | xargs %s %s" % (sstate_scan_cmd, sstate_grep_cmd, sstate_filelist_cmd, xargs_no_empty_run_cmd, sstate_sed_cmd)
574
575 bb.note("Removing hardcoded paths from sstate package: '%s'" % (sstate_hardcode_cmd))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500576 subprocess.check_output(sstate_hardcode_cmd, shell=True, cwd=sstate_builddir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500577
578 # If the fixmefn is empty, remove it..
579 if os.stat(fixmefn).st_size == 0:
580 os.remove(fixmefn)
581 else:
582 bb.note("Replacing absolute paths in fixmepath file: '%s'" % (sstate_filelist_relative_cmd))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500583 subprocess.check_output(sstate_filelist_relative_cmd, shell=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500584}
585
586def sstate_package(ss, d):
587 import oe.path
588
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500589 tmpdir = d.getVar('TMPDIR')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500590
591 sstatebuild = d.expand("${WORKDIR}/sstate-build-%s/" % ss['task'])
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500592 sstatepkg = d.getVar('SSTATE_PKG') + '_'+ ss['task'] + ".tgz"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500593 bb.utils.remove(sstatebuild, recurse=True)
594 bb.utils.mkdirhier(sstatebuild)
595 bb.utils.mkdirhier(os.path.dirname(sstatepkg))
596 for state in ss['dirs']:
597 if not os.path.exists(state[1]):
598 continue
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500599 if d.getVar('SSTATE_SKIP_CREATION') == '1':
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600600 continue
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500601 srcbase = state[0].rstrip("/").rsplit('/', 1)[0]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500602 # Find and error for absolute symlinks. We could attempt to relocate but its not
603 # clear where the symlink is relative to in this context. We could add that markup
604 # to sstate tasks but there aren't many of these so better just avoid them entirely.
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500605 for walkroot, dirs, files in os.walk(state[1]):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500606 for file in files + dirs:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500607 srcpath = os.path.join(walkroot, file)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500608 if not os.path.islink(srcpath):
609 continue
610 link = os.readlink(srcpath)
611 if not os.path.isabs(link):
612 continue
613 if not link.startswith(tmpdir):
614 continue
615 bb.error("sstate found an absolute path symlink %s pointing at %s. Please replace this with a relative link." % (srcpath, link))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500616 bb.debug(2, "Preparing tree %s for packaging at %s" % (state[1], sstatebuild + state[0]))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500617 os.rename(state[1], sstatebuild + state[0])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500618
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500619 workdir = d.getVar('WORKDIR')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500620 for plain in ss['plaindirs']:
621 pdir = plain.replace(workdir, sstatebuild)
622 bb.utils.mkdirhier(plain)
623 bb.utils.mkdirhier(pdir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500624 os.rename(plain, pdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500625
626 d.setVar('SSTATE_BUILDDIR', sstatebuild)
627 d.setVar('SSTATE_PKG', sstatepkg)
628
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500629 for f in (d.getVar('SSTATECREATEFUNCS') or '').split() + \
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500630 ['sstate_create_package', 'sstate_sign_package'] + \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500631 (d.getVar('SSTATEPOSTCREATEFUNCS') or '').split():
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500632 # All hooks should run in SSTATE_BUILDDIR.
633 bb.build.exec_func(f, d, (sstatebuild,))
634
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500635 bb.siggen.dump_this_task(sstatepkg + ".siginfo", d)
636
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500637 d.setVar('SSTATE_INSTDIR', sstatebuild)
638
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500639 return
640
641def pstaging_fetch(sstatefetch, sstatepkg, d):
642 import bb.fetch2
643
644 # Only try and fetch if the user has configured a mirror
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500645 mirrors = d.getVar('SSTATE_MIRRORS')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500646 if not mirrors:
647 return
648
649 # Copy the data object and override DL_DIR and SRC_URI
650 localdata = bb.data.createCopy(d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500651
652 dldir = localdata.expand("${SSTATE_DIR}")
653 bb.utils.mkdirhier(dldir)
654
655 localdata.delVar('MIRRORS')
656 localdata.setVar('FILESPATH', dldir)
657 localdata.setVar('DL_DIR', dldir)
658 localdata.setVar('PREMIRRORS', mirrors)
659
660 # if BB_NO_NETWORK is set but we also have SSTATE_MIRROR_ALLOW_NETWORK,
661 # we'll want to allow network access for the current set of fetches.
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500662 if localdata.getVar('BB_NO_NETWORK') == "1" and localdata.getVar('SSTATE_MIRROR_ALLOW_NETWORK') == "1":
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500663 localdata.delVar('BB_NO_NETWORK')
664
665 # Try a fetch from the sstate mirror, if it fails just return and
666 # we will build the package
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600667 uris = ['file://{0};downloadfilename={0}'.format(sstatefetch),
668 'file://{0}.siginfo;downloadfilename={0}.siginfo'.format(sstatefetch)]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500669 if bb.utils.to_boolean(d.getVar("SSTATE_VERIFY_SIG"), False):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600670 uris += ['file://{0}.sig;downloadfilename={0}.sig'.format(sstatefetch)]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500671
672 for srcuri in uris:
673 localdata.setVar('SRC_URI', srcuri)
674 try:
675 fetcher = bb.fetch2.Fetch([srcuri], localdata, cache=False)
676 fetcher.download()
677
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500678 except bb.fetch2.BBFetchException:
679 break
680
681def sstate_setscene(d):
682 shared_state = sstate_state_fromvars(d)
683 accelerate = sstate_installpkg(shared_state, d)
684 if not accelerate:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600685 bb.fatal("No suitable staging package found")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500686
687python sstate_task_prefunc () {
688 shared_state = sstate_state_fromvars(d)
689 sstate_clean(shared_state, d)
690}
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500691sstate_task_prefunc[dirs] = "${WORKDIR}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500692
693python sstate_task_postfunc () {
694 shared_state = sstate_state_fromvars(d)
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500695
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500696 for intercept in shared_state['interceptfuncs']:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500697 bb.build.exec_func(intercept, d, (d.getVar("WORKDIR"),))
698
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600699 omask = os.umask(0o002)
700 if omask != 0o002:
701 bb.note("Using umask 0o002 (not %0o) for sstate packaging" % omask)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500702 sstate_package(shared_state, d)
703 os.umask(omask)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500704
705 sstateinst = d.getVar("SSTATE_INSTDIR")
706 d.setVar('SSTATE_FIXMEDIR', shared_state['fixmedir'])
707
708 sstate_installpkgdir(shared_state, d)
709
710 bb.utils.remove(d.getVar("SSTATE_BUILDDIR"), recurse=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500711}
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500712sstate_task_postfunc[dirs] = "${WORKDIR}"
713
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500714
715#
716# Shell function to generate a sstate package from a directory
717# set as SSTATE_BUILDDIR. Will be run from within SSTATE_BUILDDIR.
718#
719sstate_create_package () {
720 TFILE=`mktemp ${SSTATE_PKG}.XXXXXXXX`
721 # Need to handle empty directories
722 if [ "$(ls -A)" ]; then
723 set +e
724 tar -czf $TFILE *
725 ret=$?
726 if [ $ret -ne 0 ] && [ $ret -ne 1 ]; then
727 exit 1
728 fi
729 set -e
730 else
731 tar -cz --file=$TFILE --files-from=/dev/null
732 fi
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500733 chmod 0664 $TFILE
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500734 mv -f $TFILE ${SSTATE_PKG}
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500735}
736
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500737python sstate_sign_package () {
738 from oe.gpg_sign import get_signer
739
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500740 if d.getVar('SSTATE_SIG_KEY'):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500741 signer = get_signer(d, 'local')
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500742 sstate_pkg = d.getVar('SSTATE_PKG')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500743 if os.path.exists(sstate_pkg + '.sig'):
744 os.unlink(sstate_pkg + '.sig')
745 signer.detach_sign(sstate_pkg, d.getVar('SSTATE_SIG_KEY', False), None,
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500746 d.getVar('SSTATE_SIG_PASSPHRASE'), armor=False)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500747}
748
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500749#
750# Shell function to decompress and prepare a package for installation
751# Will be run from within SSTATE_INSTDIR.
752#
753sstate_unpack_package () {
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500754 tar -xvzf ${SSTATE_PKG}
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500755 # update .siginfo atime on local/NFS mirror
756 [ -w ${SSTATE_PKG}.siginfo ] && [ -h ${SSTATE_PKG}.siginfo ] && touch -a ${SSTATE_PKG}.siginfo
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500757 # Use "! -w ||" to return true for read only files
758 [ ! -w ${SSTATE_PKG} ] || touch --no-dereference ${SSTATE_PKG}
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500759 [ ! -w ${SSTATE_PKG}.sig ] || [ ! -e ${SSTATE_PKG}.sig ] || touch --no-dereference ${SSTATE_PKG}.sig
760 [ ! -w ${SSTATE_PKG}.siginfo ] || [ ! -e ${SSTATE_PKG}.siginfo ] || touch --no-dereference ${SSTATE_PKG}.siginfo
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500761}
762
763BB_HASHCHECK_FUNCTION = "sstate_checkhashes"
764
765def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=False):
766
767 ret = []
768 missed = []
769 extension = ".tgz"
770 if siginfo:
771 extension = extension + ".siginfo"
772
773 def getpathcomponents(task, d):
774 # Magic data from BB_HASHFILENAME
775 splithashfn = sq_hashfn[task].split(" ")
776 spec = splithashfn[1]
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500777 if splithashfn[0] == "True":
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500778 extrapath = d.getVar("NATIVELSBSTRING") + "/"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500779 else:
780 extrapath = ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500781
782 tname = sq_task[task][3:]
783
784 if tname in ["fetch", "unpack", "patch", "populate_lic", "preconfigure"] and splithashfn[2]:
785 spec = splithashfn[2]
786 extrapath = ""
787
788 return spec, extrapath, tname
789
790
791 for task in range(len(sq_fn)):
792
793 spec, extrapath, tname = getpathcomponents(task, d)
794
795 sstatefile = d.expand("${SSTATE_DIR}/" + extrapath + generate_sstatefn(spec, sq_hash[task], d) + "_" + tname + extension)
796
797 if os.path.exists(sstatefile):
798 bb.debug(2, "SState: Found valid sstate file %s" % sstatefile)
799 ret.append(task)
800 continue
801 else:
802 missed.append(task)
803 bb.debug(2, "SState: Looked for but didn't find file %s" % sstatefile)
804
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500805 mirrors = d.getVar("SSTATE_MIRRORS")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500806 if mirrors:
807 # Copy the data object and override DL_DIR and SRC_URI
808 localdata = bb.data.createCopy(d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500809
810 dldir = localdata.expand("${SSTATE_DIR}")
811 localdata.delVar('MIRRORS')
812 localdata.setVar('FILESPATH', dldir)
813 localdata.setVar('DL_DIR', dldir)
814 localdata.setVar('PREMIRRORS', mirrors)
815
816 bb.debug(2, "SState using premirror of: %s" % mirrors)
817
818 # if BB_NO_NETWORK is set but we also have SSTATE_MIRROR_ALLOW_NETWORK,
819 # we'll want to allow network access for the current set of fetches.
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500820 if localdata.getVar('BB_NO_NETWORK') == "1" and localdata.getVar('SSTATE_MIRROR_ALLOW_NETWORK') == "1":
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500821 localdata.delVar('BB_NO_NETWORK')
822
823 from bb.fetch2 import FetchConnectionCache
824 def checkstatus_init(thread_worker):
825 thread_worker.connection_cache = FetchConnectionCache()
826
827 def checkstatus_end(thread_worker):
828 thread_worker.connection_cache.close_connections()
829
830 def checkstatus(thread_worker, arg):
831 (task, sstatefile) = arg
832
833 localdata2 = bb.data.createCopy(localdata)
834 srcuri = "file://" + sstatefile
835 localdata.setVar('SRC_URI', srcuri)
836 bb.debug(2, "SState: Attempting to fetch %s" % srcuri)
837
838 try:
839 fetcher = bb.fetch2.Fetch(srcuri.split(), localdata2,
840 connection_cache=thread_worker.connection_cache)
841 fetcher.checkstatus()
842 bb.debug(2, "SState: Successful fetch test for %s" % srcuri)
843 ret.append(task)
844 if task in missed:
845 missed.remove(task)
846 except:
847 missed.append(task)
848 bb.debug(2, "SState: Unsuccessful fetch test for %s" % srcuri)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600849 pass
850 bb.event.fire(bb.event.ProcessProgress("Checking sstate mirror object availability", len(tasklist) - thread_worker.tasks.qsize()), d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500851
852 tasklist = []
853 for task in range(len(sq_fn)):
854 if task in ret:
855 continue
856 spec, extrapath, tname = getpathcomponents(task, d)
857 sstatefile = d.expand(extrapath + generate_sstatefn(spec, sq_hash[task], d) + "_" + tname + extension)
858 tasklist.append((task, sstatefile))
859
860 if tasklist:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600861 bb.event.fire(bb.event.ProcessStarted("Checking sstate mirror object availability", len(tasklist)), d)
862
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500863 import multiprocessing
864 nproc = min(multiprocessing.cpu_count(), len(tasklist))
865
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600866 bb.event.enable_threadlock()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500867 pool = oe.utils.ThreadedPool(nproc, len(tasklist),
868 worker_init=checkstatus_init, worker_end=checkstatus_end)
869 for t in tasklist:
870 pool.add_task(checkstatus, t)
871 pool.start()
872 pool.wait_completion()
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600873 bb.event.disable_threadlock()
874
875 bb.event.fire(bb.event.ProcessFinished("Checking sstate mirror object availability"), d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500876
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500877 inheritlist = d.getVar("INHERIT")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500878 if "toaster" in inheritlist:
879 evdata = {'missed': [], 'found': []};
880 for task in missed:
881 spec, extrapath, tname = getpathcomponents(task, d)
882 sstatefile = d.expand(extrapath + generate_sstatefn(spec, sq_hash[task], d) + "_" + tname + ".tgz")
883 evdata['missed'].append( (sq_fn[task], sq_task[task], sq_hash[task], sstatefile ) )
884 for task in ret:
885 spec, extrapath, tname = getpathcomponents(task, d)
886 sstatefile = d.expand(extrapath + generate_sstatefn(spec, sq_hash[task], d) + "_" + tname + ".tgz")
887 evdata['found'].append( (sq_fn[task], sq_task[task], sq_hash[task], sstatefile ) )
888 bb.event.fire(bb.event.MetadataEvent("MissedSstate", evdata), d)
889
890 if hasattr(bb.parse.siggen, "checkhashes"):
891 bb.parse.siggen.checkhashes(missed, ret, sq_fn, sq_task, sq_hash, sq_hashfn, d)
892
893 return ret
894
895BB_SETSCENE_DEPVALID = "setscene_depvalid"
896
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500897def setscene_depvalid(task, taskdependees, notneeded, d, log=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500898 # taskdependees is a dict of tasks which depend on task, each being a 3 item list of [PN, TASKNAME, FILENAME]
899 # task is included in taskdependees too
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500900 # Return - False - We need this dependency
901 # - True - We can skip this dependency
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500902
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500903 def logit(msg, log):
904 if log is not None:
905 log.append(msg)
906 else:
907 bb.debug(2, msg)
908
909 logit("Considering setscene task: %s" % (str(taskdependees[task])), log)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500910
911 def isNativeCross(x):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500912 return x.endswith("-native") or "-cross-" in x or "-crosssdk" in x or x.endswith("-cross")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500913
914 # We only need to trigger populate_lic through direct dependencies
915 if taskdependees[task][1] == "do_populate_lic":
916 return True
917
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500918 # stash_locale and gcc_stash_builddir are never needed as a dependency for built objects
919 if taskdependees[task][1] == "do_stash_locale" or taskdependees[task][1] == "do_gcc_stash_builddir":
920 return True
921
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500922 # We only need to trigger packagedata through direct dependencies
923 # but need to preserve packagedata on packagedata links
924 if taskdependees[task][1] == "do_packagedata":
925 for dep in taskdependees:
926 if taskdependees[dep][1] == "do_packagedata":
927 return False
928 return True
929
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500930 for dep in taskdependees:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500931 logit(" considering dependency: %s" % (str(taskdependees[dep])), log)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500932 if task == dep:
933 continue
934 if dep in notneeded:
935 continue
936 # do_package_write_* and do_package doesn't need do_package
937 if taskdependees[task][1] == "do_package" and taskdependees[dep][1] in ['do_package', 'do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm', 'do_packagedata', 'do_package_qa']:
938 continue
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500939 # do_package_write_* need do_populate_sysroot as they're mainly postinstall dependencies
940 if taskdependees[task][1] == "do_populate_sysroot" and taskdependees[dep][1] in ['do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm']:
941 return False
942 # do_package/packagedata/package_qa don't need do_populate_sysroot
943 if taskdependees[task][1] == "do_populate_sysroot" and taskdependees[dep][1] in ['do_package', 'do_packagedata', 'do_package_qa']:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500944 continue
945 # Native/Cross packages don't exist and are noexec anyway
946 if isNativeCross(taskdependees[dep][0]) and taskdependees[dep][1] in ['do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm', 'do_packagedata', 'do_package', 'do_package_qa']:
947 continue
948
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500949 # This is due to the [depends] in useradd.bbclass complicating matters
950 # The logic *is* reversed here due to the way hard setscene dependencies are injected
951 if (taskdependees[task][1] == 'do_package' or taskdependees[task][1] == 'do_populate_sysroot') and taskdependees[dep][0].endswith(('shadow-native', 'shadow-sysroot', 'base-passwd', 'pseudo-native')) and taskdependees[dep][1] == 'do_populate_sysroot':
952 continue
953
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500954 # Consider sysroot depending on sysroot tasks
955 if taskdependees[task][1] == 'do_populate_sysroot' and taskdependees[dep][1] == 'do_populate_sysroot':
956 # base-passwd/shadow-sysroot don't need their dependencies
957 if taskdependees[dep][0].endswith(("base-passwd", "shadow-sysroot")):
958 continue
959 # Nothing need depend on libc-initial/gcc-cross-initial
960 if "-initial" in taskdependees[task][0]:
961 continue
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600962 # For meta-extsdk-toolchain we want all sysroot dependencies
963 if taskdependees[dep][0] == 'meta-extsdk-toolchain':
964 return False
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500965 # Native/Cross populate_sysroot need their dependencies
966 if isNativeCross(taskdependees[task][0]) and isNativeCross(taskdependees[dep][0]):
967 return False
968 # Target populate_sysroot depended on by cross tools need to be installed
969 if isNativeCross(taskdependees[dep][0]):
970 return False
971 # Native/cross tools depended upon by target sysroot are not needed
972 if isNativeCross(taskdependees[task][0]):
973 continue
974 # Target populate_sysroot need their dependencies
975 return False
976
977 if taskdependees[task][1] == 'do_shared_workdir':
978 continue
979
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500980 if taskdependees[dep][1] == "do_populate_lic":
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500981 continue
982
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500983
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500984 # Safe fallthrough default
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500985 logit(" Default setscene dependency fall through due to dependency: %s" % (str(taskdependees[dep])), log)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500986 return False
987 return True
988
989addhandler sstate_eventhandler
990sstate_eventhandler[eventmask] = "bb.build.TaskSucceeded"
991python sstate_eventhandler() {
992 d = e.data
993 # When we write an sstate package we rewrite the SSTATE_PKG
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500994 spkg = d.getVar('SSTATE_PKG')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500995 if not spkg.endswith(".tgz"):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500996 taskname = d.getVar("BB_RUNTASK")[3:]
997 spec = d.getVar('SSTATE_PKGSPEC')
998 swspec = d.getVar('SSTATE_SWSPEC')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500999 if taskname in ["fetch", "unpack", "patch", "populate_lic", "preconfigure"] and swspec:
1000 d.setVar("SSTATE_PKGSPEC", "${SSTATE_SWSPEC}")
1001 d.setVar("SSTATE_EXTRAPATH", "")
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001002 sstatepkg = d.getVar('SSTATE_PKG')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001003 bb.siggen.dump_this_task(sstatepkg + '_' + taskname + ".tgz" ".siginfo", d)
1004}
1005
1006SSTATE_PRUNE_OBSOLETEWORKDIR = "1"
1007
1008# Event handler which removes manifests and stamps file for
1009# recipes which are no longer reachable in a build where they
1010# once were.
1011# Also optionally removes the workdir of those tasks/recipes
1012#
1013addhandler sstate_eventhandler2
1014sstate_eventhandler2[eventmask] = "bb.event.ReachableStamps"
1015python sstate_eventhandler2() {
1016 import glob
1017 d = e.data
1018 stamps = e.stamps.values()
1019 removeworkdir = (d.getVar("SSTATE_PRUNE_OBSOLETEWORKDIR", False) == "1")
1020 seen = []
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001021 for a in d.getVar("SSTATE_ARCHS").split():
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001022 toremove = []
1023 i = d.expand("${SSTATE_MANIFESTS}/index-" + a)
1024 if not os.path.exists(i):
1025 continue
1026 with open(i, "r") as f:
1027 lines = f.readlines()
1028 for l in lines:
1029 (stamp, manifest, workdir) = l.split()
1030 if stamp not in stamps:
1031 toremove.append(l)
1032 if stamp not in seen:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001033 bb.debug(2, "Stamp %s is not reachable, removing related manifests" % stamp)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001034 seen.append(stamp)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001035
1036 if toremove:
1037 bb.note("There are %d recipes to be removed from sysroot %s, removing..." % (len(toremove), a))
1038
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001039 for r in toremove:
1040 (stamp, manifest, workdir) = r.split()
1041 for m in glob.glob(manifest + ".*"):
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001042 if m.endswith(".postrm"):
1043 continue
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001044 sstate_clean_manifest(m, d)
1045 bb.utils.remove(stamp + "*")
1046 if removeworkdir:
1047 bb.utils.remove(workdir, recurse = True)
1048 lines.remove(r)
1049 with open(i, "w") as f:
1050 for l in lines:
1051 f.write(l)
1052}