blob: 9fc8f4f2839b284ff6b02fbd10e7f71531ba86d5 [file] [log] [blame]
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001# These directories will be staged in the sysroot
2SYSROOT_DIRS = " \
3 ${includedir} \
4 ${libdir} \
5 ${base_libdir} \
6 ${nonarch_base_libdir} \
7 ${datadir} \
Andrew Geisslerd1e89492021-02-12 15:35:20 -06008 /sysroot-only \
Patrick Williamsc0f7c042017-02-23 20:41:17 -06009"
10
11# These directories are also staged in the sysroot when they contain files that
12# are usable on the build system
13SYSROOT_DIRS_NATIVE = " \
14 ${bindir} \
15 ${sbindir} \
16 ${base_bindir} \
17 ${base_sbindir} \
18 ${libexecdir} \
19 ${sysconfdir} \
20 ${localstatedir} \
21"
Patrick Williams213cb262021-08-07 19:21:33 -050022SYSROOT_DIRS:append:class-native = " ${SYSROOT_DIRS_NATIVE}"
23SYSROOT_DIRS:append:class-cross = " ${SYSROOT_DIRS_NATIVE}"
24SYSROOT_DIRS:append:class-crosssdk = " ${SYSROOT_DIRS_NATIVE}"
Patrick Williamsc0f7c042017-02-23 20:41:17 -060025
26# These directories will not be staged in the sysroot
Andrew Geissler7e0e3c02022-02-25 20:34:39 +000027SYSROOT_DIRS_IGNORE = " \
Patrick Williamsc0f7c042017-02-23 20:41:17 -060028 ${mandir} \
29 ${docdir} \
30 ${infodir} \
Andrew Geisslerd1e89492021-02-12 15:35:20 -060031 ${datadir}/X11/locale \
Patrick Williamsc0f7c042017-02-23 20:41:17 -060032 ${datadir}/applications \
Andrew Geisslerd1e89492021-02-12 15:35:20 -060033 ${datadir}/bash-completion \
Patrick Williamsc0f7c042017-02-23 20:41:17 -060034 ${datadir}/fonts \
Brad Bishopc342db32019-05-15 21:57:59 -040035 ${datadir}/gtk-doc/html \
Andrew Geisslerd1e89492021-02-12 15:35:20 -060036 ${datadir}/installed-tests \
Brad Bishopc342db32019-05-15 21:57:59 -040037 ${datadir}/locale \
Patrick Williamsc0f7c042017-02-23 20:41:17 -060038 ${datadir}/pixmaps \
Andrew Geisslerd1e89492021-02-12 15:35:20 -060039 ${datadir}/terminfo \
Andrew Geissler82c905d2020-04-13 13:39:40 -050040 ${libdir}/${BPN}/ptest \
Patrick Williamsc0f7c042017-02-23 20:41:17 -060041"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050042
43sysroot_stage_dir() {
44 src="$1"
45 dest="$2"
46 # if the src doesn't exist don't do anything
47 if [ ! -d "$src" ]; then
48 return
49 fi
50
51 mkdir -p "$dest"
Andrew Geissler7e0e3c02022-02-25 20:34:39 +000052 rdest=$(realpath --relative-to="$src" "$dest")
Patrick Williamsc124f4f2015-09-15 14:41:29 -050053 (
54 cd $src
Andrew Geissler7e0e3c02022-02-25 20:34:39 +000055 find . -print0 | cpio --null -pdlu $rdest
Patrick Williamsc124f4f2015-09-15 14:41:29 -050056 )
57}
58
Patrick Williamsc124f4f2015-09-15 14:41:29 -050059sysroot_stage_dirs() {
60 from="$1"
61 to="$2"
62
Patrick Williamsc0f7c042017-02-23 20:41:17 -060063 for dir in ${SYSROOT_DIRS}; do
64 sysroot_stage_dir "$from$dir" "$to$dir"
65 done
66
67 # Remove directories we do not care about
Andrew Geissler7e0e3c02022-02-25 20:34:39 +000068 for dir in ${SYSROOT_DIRS_IGNORE}; do
Patrick Williamsc0f7c042017-02-23 20:41:17 -060069 rm -rf "$to$dir"
70 done
Patrick Williamsc124f4f2015-09-15 14:41:29 -050071}
72
73sysroot_stage_all() {
74 sysroot_stage_dirs ${D} ${SYSROOT_DESTDIR}
75}
76
77python sysroot_strip () {
Brad Bishopd7bf8c12018-02-25 22:55:05 -050078 inhibit_sysroot = d.getVar('INHIBIT_SYSROOT_STRIP')
79 if inhibit_sysroot and oe.types.boolean(inhibit_sysroot):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080080 return
Patrick Williamsc124f4f2015-09-15 14:41:29 -050081
Brad Bishopd7bf8c12018-02-25 22:55:05 -050082 dstdir = d.getVar('SYSROOT_DESTDIR')
Brad Bishop6e60e8b2018-02-01 10:27:11 -050083 pn = d.getVar('PN')
Andrew Geissler82c905d2020-04-13 13:39:40 -050084 libdir = d.getVar("libdir")
85 base_libdir = d.getVar("base_libdir")
Patrick Williams213cb262021-08-07 19:21:33 -050086 qa_already_stripped = 'already-stripped' in (d.getVar('INSANE_SKIP:' + pn) or "").split()
Brad Bishopd7bf8c12018-02-25 22:55:05 -050087 strip_cmd = d.getVar("STRIP")
Patrick Williamsc124f4f2015-09-15 14:41:29 -050088
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080089 oe.package.strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, d,
Brad Bishopd7bf8c12018-02-25 22:55:05 -050090 qa_already_stripped=qa_already_stripped)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050091}
92
93do_populate_sysroot[dirs] = "${SYSROOT_DESTDIR}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050094
95addtask populate_sysroot after do_install
96
97SYSROOT_PREPROCESS_FUNCS ?= ""
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050098SYSROOT_DESTDIR = "${WORKDIR}/sysroot-destdir"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050099
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500100python do_populate_sysroot () {
Andrew Geissler4b740dc2020-05-05 08:54:39 -0500101 # SYSROOT 'version' 2
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500102 bb.build.exec_func("sysroot_stage_all", d)
103 bb.build.exec_func("sysroot_strip", d)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500104 for f in (d.getVar('SYSROOT_PREPROCESS_FUNCS') or '').split():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500105 bb.build.exec_func(f, d)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500106 pn = d.getVar("PN")
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000107 multiprov = d.getVar("BB_MULTI_PROVIDER_ALLOWED").split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500108 provdir = d.expand("${SYSROOT_DESTDIR}${base_prefix}/sysroot-providers/")
109 bb.utils.mkdirhier(provdir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500110 for p in d.getVar("PROVIDES").split():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500111 if p in multiprov:
112 continue
113 p = p.replace("/", "_")
114 with open(provdir + p, "w") as f:
115 f.write(pn)
116}
117
118do_populate_sysroot[vardeps] += "${SYSROOT_PREPROCESS_FUNCS}"
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000119do_populate_sysroot[vardepsexclude] += "BB_MULTI_PROVIDER_ALLOWED"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500120
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500121POPULATESYSROOTDEPS = ""
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000122POPULATESYSROOTDEPS:class-target = "virtual/${MLPREFIX}${HOST_PREFIX}binutils:do_populate_sysroot"
123POPULATESYSROOTDEPS:class-nativesdk = "virtual/${HOST_PREFIX}binutils-crosssdk:do_populate_sysroot"
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500124do_populate_sysroot[depends] += "${POPULATESYSROOTDEPS}"
125
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500126SSTATETASKS += "do_populate_sysroot"
127do_populate_sysroot[cleandirs] = "${SYSROOT_DESTDIR}"
128do_populate_sysroot[sstate-inputdirs] = "${SYSROOT_DESTDIR}"
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500129do_populate_sysroot[sstate-outputdirs] = "${COMPONENTS_DIR}/${PACKAGE_ARCH}/${PN}"
130do_populate_sysroot[sstate-fixmedir] = "${COMPONENTS_DIR}/${PACKAGE_ARCH}/${PN}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500131
132python do_populate_sysroot_setscene () {
133 sstate_setscene(d)
134}
135addtask do_populate_sysroot_setscene
136
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500137def staging_copyfile(c, target, dest, postinsts, seendirs):
138 import errno
139
140 destdir = os.path.dirname(dest)
141 if destdir not in seendirs:
142 bb.utils.mkdirhier(destdir)
143 seendirs.add(destdir)
144 if "/usr/bin/postinst-" in c:
145 postinsts.append(dest)
146 if os.path.islink(c):
147 linkto = os.readlink(c)
148 if os.path.lexists(dest):
149 if not os.path.islink(dest):
150 raise OSError(errno.EEXIST, "Link %s already exists as a file" % dest, dest)
151 if os.readlink(dest) == linkto:
152 return dest
153 raise OSError(errno.EEXIST, "Link %s already exists to a different location? (%s vs %s)" % (dest, os.readlink(dest), linkto), dest)
154 os.symlink(linkto, dest)
155 #bb.warn(c)
156 else:
157 try:
158 os.link(c, dest)
159 except OSError as err:
160 if err.errno == errno.EXDEV:
161 bb.utils.copyfile(c, dest)
162 else:
163 raise
164 return dest
165
166def staging_copydir(c, target, dest, seendirs):
167 if dest not in seendirs:
168 bb.utils.mkdirhier(dest)
169 seendirs.add(dest)
170
171def staging_processfixme(fixme, target, recipesysroot, recipesysrootnative, d):
172 import subprocess
173
174 if not fixme:
175 return
176 cmd = "sed -e 's:^[^/]*/:%s/:g' %s | xargs sed -i -e 's:FIXMESTAGINGDIRTARGET:%s:g; s:FIXMESTAGINGDIRHOST:%s:g'" % (target, " ".join(fixme), recipesysroot, recipesysrootnative)
Brad Bishop15ae2502019-06-18 21:44:24 -0400177 for fixmevar in ['PSEUDO_SYSROOT', 'HOSTTOOLS_DIR', 'PKGDATA_DIR', 'PSEUDO_LOCALSTATEDIR', 'LOGFIFO']:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500178 fixme_path = d.getVar(fixmevar)
179 cmd += " -e 's:FIXME_%s:%s:g'" % (fixmevar, fixme_path)
180 bb.debug(2, cmd)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400181 subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500182
183
184def staging_populate_sysroot_dir(targetsysroot, nativesysroot, native, d):
185 import glob
186 import subprocess
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500187 import errno
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500188
189 fixme = []
190 postinsts = []
191 seendirs = set()
192 stagingdir = d.getVar("STAGING_DIR")
193 if native:
194 pkgarchs = ['${BUILD_ARCH}', '${BUILD_ARCH}_*']
195 targetdir = nativesysroot
196 else:
197 pkgarchs = ['${MACHINE_ARCH}']
198 pkgarchs = pkgarchs + list(reversed(d.getVar("PACKAGE_EXTRA_ARCHS").split()))
199 pkgarchs.append('allarch')
200 targetdir = targetsysroot
201
202 bb.utils.mkdirhier(targetdir)
203 for pkgarch in pkgarchs:
204 for manifest in glob.glob(d.expand("${SSTATE_MANIFESTS}/manifest-%s-*.populate_sysroot" % pkgarch)):
205 if manifest.endswith("-initial.populate_sysroot"):
Brad Bishop79641f22019-09-10 07:20:22 -0400206 # skip libgcc-initial due to file overlap
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500207 continue
Brad Bishop19323692019-04-05 15:28:33 -0400208 if not native and (manifest.endswith("-native.populate_sysroot") or "nativesdk-" in manifest):
209 continue
210 if native and not (manifest.endswith("-native.populate_sysroot") or manifest.endswith("-cross.populate_sysroot") or "-cross-" in manifest):
211 continue
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500212 tmanifest = targetdir + "/" + os.path.basename(manifest)
213 if os.path.exists(tmanifest):
214 continue
215 try:
216 os.link(manifest, tmanifest)
217 except OSError as err:
218 if err.errno == errno.EXDEV:
219 bb.utils.copyfile(manifest, tmanifest)
220 else:
221 raise
222 with open(manifest, "r") as f:
223 for l in f:
224 l = l.strip()
225 if l.endswith("/fixmepath"):
226 fixme.append(l)
227 continue
228 if l.endswith("/fixmepath.cmd"):
229 continue
230 dest = l.replace(stagingdir, "")
231 dest = targetdir + "/" + "/".join(dest.split("/")[3:])
232 if l.endswith("/"):
233 staging_copydir(l, targetdir, dest, seendirs)
234 continue
235 try:
236 staging_copyfile(l, targetdir, dest, postinsts, seendirs)
237 except FileExistsError:
238 continue
239
240 staging_processfixme(fixme, targetdir, targetsysroot, nativesysroot, d)
241 for p in postinsts:
Brad Bishop316dfdd2018-06-25 12:45:53 -0400242 subprocess.check_output(p, shell=True, stderr=subprocess.STDOUT)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500243
244#
245# Manifests here are complicated. The main sysroot area has the unpacked sstate
246# which us unrelocated and tracked by the main sstate manifests. Each recipe
247# specific sysroot has manifests for each dependency that is installed there.
248# The task hash is used to tell whether the data needs to be reinstalled. We
249# use a symlink to point to the currently installed hash. There is also a
250# "complete" stamp file which is used to mark if installation completed. If
251# something fails (e.g. a postinst), this won't get written and we would
252# remove and reinstall the dependency. This also means partially installed
253# dependencies should get cleaned up correctly.
254#
255
256python extend_recipe_sysroot() {
257 import copy
258 import subprocess
259 import errno
260 import collections
261 import glob
262
263 taskdepdata = d.getVar("BB_TASKDEPDATA", False)
264 mytaskname = d.getVar("BB_RUNTASK")
265 if mytaskname.endswith("_setscene"):
266 mytaskname = mytaskname.replace("_setscene", "")
267 workdir = d.getVar("WORKDIR")
268 #bb.warn(str(taskdepdata))
269 pn = d.getVar("PN")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500270 stagingdir = d.getVar("STAGING_DIR")
271 sharedmanifests = d.getVar("COMPONENTS_DIR") + "/manifests"
272 recipesysroot = d.getVar("RECIPE_SYSROOT")
273 recipesysrootnative = d.getVar("RECIPE_SYSROOT_NATIVE")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500274
275 # Detect bitbake -b usage
276 nodeps = d.getVar("BB_LIMITEDDEPS") or False
277 if nodeps:
278 lock = bb.utils.lockfile(recipesysroot + "/sysroot.lock")
279 staging_populate_sysroot_dir(recipesysroot, recipesysrootnative, True, d)
280 staging_populate_sysroot_dir(recipesysroot, recipesysrootnative, False, d)
281 bb.utils.unlockfile(lock)
282 return
283
284 start = None
285 configuredeps = []
Andrew Geissler82c905d2020-04-13 13:39:40 -0500286 owntaskdeps = []
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500287 for dep in taskdepdata:
288 data = taskdepdata[dep]
289 if data[1] == mytaskname and data[0] == pn:
290 start = dep
Andrew Geissler82c905d2020-04-13 13:39:40 -0500291 elif data[0] == pn:
292 owntaskdeps.append(data[1])
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500293 if start is None:
294 bb.fatal("Couldn't find ourself in BB_TASKDEPDATA?")
295
296 # We need to figure out which sysroot files we need to expose to this task.
297 # This needs to match what would get restored from sstate, which is controlled
298 # ultimately by calls from bitbake to setscene_depvalid().
299 # That function expects a setscene dependency tree. We build a dependency tree
300 # condensed to inter-sstate task dependencies, similar to that used by setscene
301 # tasks. We can then call into setscene_depvalid() and decide
302 # which dependencies we can "see" and should expose in the recipe specific sysroot.
303 setscenedeps = copy.deepcopy(taskdepdata)
304
305 start = set([start])
306
307 sstatetasks = d.getVar("SSTATETASKS").split()
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800308 # Add recipe specific tasks referenced by setscene_depvalid()
309 sstatetasks.append("do_stash_locale")
Andrew Geissler5199d832021-09-24 16:47:35 -0500310 sstatetasks.append("do_deploy")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500311
312 def print_dep_tree(deptree):
313 data = ""
314 for dep in deptree:
315 deps = " " + "\n ".join(deptree[dep][3]) + "\n"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800316 data = data + "%s:\n %s\n %s\n%s %s\n %s\n" % (deptree[dep][0], deptree[dep][1], deptree[dep][2], deps, deptree[dep][4], deptree[dep][5])
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500317 return data
318
319 #bb.note("Full dep tree is:\n%s" % print_dep_tree(taskdepdata))
320
321 #bb.note(" start2 is %s" % str(start))
322
323 # If start is an sstate task (like do_package) we need to add in its direct dependencies
324 # else the code below won't recurse into them.
325 for dep in set(start):
326 for dep2 in setscenedeps[dep][3]:
327 start.add(dep2)
328 start.remove(dep)
329
330 #bb.note(" start3 is %s" % str(start))
331
332 # Create collapsed do_populate_sysroot -> do_populate_sysroot tree
333 for dep in taskdepdata:
334 data = setscenedeps[dep]
335 if data[1] not in sstatetasks:
336 for dep2 in setscenedeps:
337 data2 = setscenedeps[dep2]
338 if dep in data2[3]:
339 data2[3].update(setscenedeps[dep][3])
340 data2[3].remove(dep)
341 if dep in start:
342 start.update(setscenedeps[dep][3])
343 start.remove(dep)
344 del setscenedeps[dep]
345
346 # Remove circular references
347 for dep in setscenedeps:
348 if dep in setscenedeps[dep][3]:
349 setscenedeps[dep][3].remove(dep)
350
351 #bb.note("Computed dep tree is:\n%s" % print_dep_tree(setscenedeps))
352 #bb.note(" start is %s" % str(start))
353
354 # Direct dependencies should be present and can be depended upon
355 for dep in set(start):
356 if setscenedeps[dep][1] == "do_populate_sysroot":
357 if dep not in configuredeps:
358 configuredeps.append(dep)
359 bb.note("Direct dependencies are %s" % str(configuredeps))
360 #bb.note(" or %s" % str(start))
361
362 msgbuf = []
363 # Call into setscene_depvalid for each sub-dependency and only copy sysroot files
364 # for ones that would be restored from sstate.
365 done = list(start)
366 next = list(start)
367 while next:
368 new = []
369 for dep in next:
370 data = setscenedeps[dep]
371 for datadep in data[3]:
372 if datadep in done:
373 continue
374 taskdeps = {}
375 taskdeps[dep] = setscenedeps[dep][:2]
376 taskdeps[datadep] = setscenedeps[datadep][:2]
377 retval = setscene_depvalid(datadep, taskdeps, [], d, msgbuf)
378 if retval:
379 msgbuf.append("Skipping setscene dependency %s for installation into the sysroot" % datadep)
380 continue
381 done.append(datadep)
382 new.append(datadep)
383 if datadep not in configuredeps and setscenedeps[datadep][1] == "do_populate_sysroot":
384 configuredeps.append(datadep)
385 msgbuf.append("Adding dependency on %s" % setscenedeps[datadep][0])
386 else:
387 msgbuf.append("Following dependency on %s" % setscenedeps[datadep][0])
388 next = new
389
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500390 # This logging is too verbose for day to day use sadly
391 #bb.debug(2, "\n".join(msgbuf))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500392
393 depdir = recipesysrootnative + "/installeddeps"
394 bb.utils.mkdirhier(depdir)
395 bb.utils.mkdirhier(sharedmanifests)
396
397 lock = bb.utils.lockfile(recipesysroot + "/sysroot.lock")
398
399 fixme = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500400 seendirs = set()
401 postinsts = []
402 multilibs = {}
403 manifests = {}
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500404 # All files that we're going to be installing, to find conflicts.
405 fileset = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500406
407 for f in os.listdir(depdir):
408 if not f.endswith(".complete"):
409 continue
410 f = depdir + "/" + f
411 if os.path.islink(f) and not os.path.exists(f):
412 bb.note("%s no longer exists, removing from sysroot" % f)
413 lnk = os.readlink(f.replace(".complete", ""))
Andrew Geissler5f350902021-07-23 13:09:54 -0400414 sstate_clean_manifest(depdir + "/" + lnk, d, canrace=True, prefix=workdir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500415 os.unlink(f)
416 os.unlink(f.replace(".complete", ""))
417
418 installed = []
419 for dep in configuredeps:
420 c = setscenedeps[dep][0]
421 if mytaskname in ["do_sdk_depends", "do_populate_sdk_ext"] and c.endswith("-initial"):
422 bb.note("Skipping initial setscene dependency %s for installation into the sysroot" % c)
423 continue
424 installed.append(c)
425
426 # We want to remove anything which this task previously installed but is no longer a dependency
427 taskindex = depdir + "/" + "index." + mytaskname
428 if os.path.exists(taskindex):
429 potential = []
430 with open(taskindex, "r") as f:
431 for l in f:
432 l = l.strip()
433 if l not in installed:
434 fl = depdir + "/" + l
435 if not os.path.exists(fl):
436 # Was likely already uninstalled
437 continue
438 potential.append(l)
Andrew Geissler82c905d2020-04-13 13:39:40 -0500439 # We need to ensure no other task needs this dependency. We hold the sysroot
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500440 # lock so we ca search the indexes to check
441 if potential:
442 for i in glob.glob(depdir + "/index.*"):
443 if i.endswith("." + mytaskname):
444 continue
445 with open(i, "r") as f:
446 for l in f:
Andrew Geissler82c905d2020-04-13 13:39:40 -0500447 if l.startswith("TaskDeps:"):
448 prevtasks = l.split()[1:]
449 if mytaskname in prevtasks:
450 # We're a dependency of this task so we can clear items out the sysroot
451 break
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500452 l = l.strip()
453 if l in potential:
454 potential.remove(l)
455 for l in potential:
456 fl = depdir + "/" + l
457 bb.note("Task %s no longer depends on %s, removing from sysroot" % (mytaskname, l))
458 lnk = os.readlink(fl)
Andrew Geissler5f350902021-07-23 13:09:54 -0400459 sstate_clean_manifest(depdir + "/" + lnk, d, canrace=True, prefix=workdir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500460 os.unlink(fl)
461 os.unlink(fl + ".complete")
462
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500463 msg_exists = []
464 msg_adding = []
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800465
Brad Bishop1d80a2e2019-11-15 16:35:03 -0500466 # Handle all removals first since files may move between recipes
467 for dep in configuredeps:
468 c = setscenedeps[dep][0]
469 if c not in installed:
470 continue
471 taskhash = setscenedeps[dep][5]
472 taskmanifest = depdir + "/" + c + "." + taskhash
473
474 if os.path.exists(depdir + "/" + c):
475 lnk = os.readlink(depdir + "/" + c)
476 if lnk == c + "." + taskhash and os.path.exists(depdir + "/" + c + ".complete"):
477 continue
478 else:
479 bb.note("%s exists in sysroot, but is stale (%s vs. %s), removing." % (c, lnk, c + "." + taskhash))
Andrew Geissler5f350902021-07-23 13:09:54 -0400480 sstate_clean_manifest(depdir + "/" + lnk, d, canrace=True, prefix=workdir)
Brad Bishop1d80a2e2019-11-15 16:35:03 -0500481 os.unlink(depdir + "/" + c)
482 if os.path.lexists(depdir + "/" + c + ".complete"):
483 os.unlink(depdir + "/" + c + ".complete")
484 elif os.path.lexists(depdir + "/" + c):
485 os.unlink(depdir + "/" + c)
486
Andrew Geissler82c905d2020-04-13 13:39:40 -0500487 binfiles = {}
Brad Bishop1d80a2e2019-11-15 16:35:03 -0500488 # Now handle installs
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500489 for dep in configuredeps:
490 c = setscenedeps[dep][0]
491 if c not in installed:
492 continue
493 taskhash = setscenedeps[dep][5]
494 taskmanifest = depdir + "/" + c + "." + taskhash
495
496 if os.path.exists(depdir + "/" + c):
497 lnk = os.readlink(depdir + "/" + c)
498 if lnk == c + "." + taskhash and os.path.exists(depdir + "/" + c + ".complete"):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500499 msg_exists.append(c)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500500 continue
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500501
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500502 msg_adding.append(c)
503
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500504 os.symlink(c + "." + taskhash, depdir + "/" + c)
505
Brad Bishop316dfdd2018-06-25 12:45:53 -0400506 manifest, d2 = oe.sstatesig.find_sstate_manifest(c, setscenedeps[dep][2], "populate_sysroot", d, multilibs)
Brad Bishopd5ae7d92018-06-14 09:52:03 -0700507 if d2 is not d:
508 # If we don't do this, the recipe sysroot will be placed in the wrong WORKDIR for multilibs
509 # We need a consistent WORKDIR for the image
510 d2.setVar("WORKDIR", d.getVar("WORKDIR"))
Brad Bishop316dfdd2018-06-25 12:45:53 -0400511 destsysroot = d2.getVar("RECIPE_SYSROOT")
Brad Bishopd5ae7d92018-06-14 09:52:03 -0700512 # We put allarch recipes into the default sysroot
513 if manifest and "allarch" in manifest:
514 destsysroot = d.getVar("RECIPE_SYSROOT")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500515
516 native = False
Brad Bishop316dfdd2018-06-25 12:45:53 -0400517 if c.endswith("-native") or "-cross-" in c or "-crosssdk" in c:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500518 native = True
Brad Bishop316dfdd2018-06-25 12:45:53 -0400519
520 if manifest:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500521 newmanifest = collections.OrderedDict()
Brad Bishopd5ae7d92018-06-14 09:52:03 -0700522 targetdir = destsysroot
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500523 if native:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500524 targetdir = recipesysrootnative
Brad Bishopd5ae7d92018-06-14 09:52:03 -0700525 if targetdir not in fixme:
526 fixme[targetdir] = []
527 fm = fixme[targetdir]
528
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500529 with open(manifest, "r") as f:
530 manifests[dep] = manifest
531 for l in f:
532 l = l.strip()
533 if l.endswith("/fixmepath"):
534 fm.append(l)
535 continue
536 if l.endswith("/fixmepath.cmd"):
537 continue
538 dest = l.replace(stagingdir, "")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500539 dest = "/" + "/".join(dest.split("/")[3:])
540 newmanifest[l] = targetdir + dest
541
542 # Check if files have already been installed by another
543 # recipe and abort if they have, explaining what recipes are
544 # conflicting.
545 hashname = targetdir + dest
546 if not hashname.endswith("/"):
547 if hashname in fileset:
548 bb.fatal("The file %s is installed by both %s and %s, aborting" % (dest, c, fileset[hashname]))
549 else:
550 fileset[hashname] = c
551
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500552 # Having multiple identical manifests in each sysroot eats diskspace so
553 # create a shared pool of them and hardlink if we can.
554 # We create the manifest in advance so that if something fails during installation,
555 # or the build is interrupted, subsequent exeuction can cleanup.
556 sharedm = sharedmanifests + "/" + os.path.basename(taskmanifest)
557 if not os.path.exists(sharedm):
558 smlock = bb.utils.lockfile(sharedm + ".lock")
559 # Can race here. You'd think it just means we may not end up with all copies hardlinked to each other
560 # but python can lose file handles so we need to do this under a lock.
561 if not os.path.exists(sharedm):
562 with open(sharedm, 'w') as m:
563 for l in newmanifest:
564 dest = newmanifest[l]
565 m.write(dest.replace(workdir + "/", "") + "\n")
566 bb.utils.unlockfile(smlock)
567 try:
568 os.link(sharedm, taskmanifest)
569 except OSError as err:
570 if err.errno == errno.EXDEV:
571 bb.utils.copyfile(sharedm, taskmanifest)
572 else:
573 raise
574 # Finally actually install the files
575 for l in newmanifest:
576 dest = newmanifest[l]
577 if l.endswith("/"):
578 staging_copydir(l, targetdir, dest, seendirs)
579 continue
Andrew Geissler82c905d2020-04-13 13:39:40 -0500580 if "/bin/" in l or "/sbin/" in l:
581 # defer /*bin/* files until last in case they need libs
582 binfiles[l] = (targetdir, dest)
583 else:
584 staging_copyfile(l, targetdir, dest, postinsts, seendirs)
585
586 # Handle deferred binfiles
587 for l in binfiles:
588 (targetdir, dest) = binfiles[l]
589 staging_copyfile(l, targetdir, dest, postinsts, seendirs)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500590
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500591 bb.note("Installed into sysroot: %s" % str(msg_adding))
592 bb.note("Skipping as already exists in sysroot: %s" % str(msg_exists))
593
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500594 for f in fixme:
Brad Bishopd5ae7d92018-06-14 09:52:03 -0700595 staging_processfixme(fixme[f], f, recipesysroot, recipesysrootnative, d)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500596
597 for p in postinsts:
Brad Bishop316dfdd2018-06-25 12:45:53 -0400598 subprocess.check_output(p, shell=True, stderr=subprocess.STDOUT)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500599
600 for dep in manifests:
601 c = setscenedeps[dep][0]
602 os.symlink(manifests[dep], depdir + "/" + c + ".complete")
603
604 with open(taskindex, "w") as f:
Andrew Geissler82c905d2020-04-13 13:39:40 -0500605 f.write("TaskDeps: " + " ".join(owntaskdeps) + "\n")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500606 for l in sorted(installed):
607 f.write(l + "\n")
608
609 bb.utils.unlockfile(lock)
610}
611extend_recipe_sysroot[vardepsexclude] += "MACHINE_ARCH PACKAGE_EXTRA_ARCHS SDK_ARCH BUILD_ARCH SDK_OS BB_TASKDEPDATA"
612
Brad Bishop19323692019-04-05 15:28:33 -0400613do_prepare_recipe_sysroot[deptask] = "do_populate_sysroot"
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500614python do_prepare_recipe_sysroot () {
615 bb.build.exec_func("extend_recipe_sysroot", d)
616}
617addtask do_prepare_recipe_sysroot before do_configure after do_fetch
618
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500619python staging_taskhandler() {
620 bbtasks = e.tasklist
621 for task in bbtasks:
622 deps = d.getVarFlag(task, "depends")
Andrew Geisslerc3d88e42020-10-02 09:45:00 -0500623 if task == "do_configure" or (deps and "populate_sysroot" in deps):
Andrew Geisslereff27472021-10-29 15:35:00 -0500624 d.prependVarFlag(task, "prefuncs", "extend_recipe_sysroot ")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500625}
626staging_taskhandler[eventmask] = "bb.event.RecipeTaskPreProcess"
627addhandler staging_taskhandler
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000628
629
630#
631# Target build output, stored in do_populate_sysroot or do_package can depend
632# not only upon direct dependencies but also indirect ones. A good example is
633# linux-libc-headers. The toolchain depends on this but most target recipes do
634# not. There are some headers which are not used by the toolchain build and do
635# not change the toolchain task output, hence the task hashes can change without
636# changing the sysroot output of that recipe yet they can influence others.
637#
638# A specific example is rtc.h which can change rtcwake.c in util-linux but is not
639# used in the glibc or gcc build. To account for this, we need to account for the
640# populate_sysroot hashes in the task output hashes.
641#
642python target_add_sysroot_deps () {
643 current_task = "do_" + d.getVar("BB_CURRENTTASK")
644 if current_task not in ["do_populate_sysroot", "do_package"]:
645 return
646
647 pn = d.getVar("PN")
648 if pn.endswith("-native"):
649 return
650
651 taskdepdata = d.getVar("BB_TASKDEPDATA", False)
652 deps = {}
653 for dep in taskdepdata.values():
Patrick Williams03907ee2022-05-01 06:28:52 -0500654 if dep[1] == "do_populate_sysroot" and not dep[0].endswith(("-native", "-initial")) and "-cross-" not in dep[0] and dep[0] != pn:
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000655 deps[dep[0]] = dep[6]
656
657 d.setVar("HASHEQUIV_EXTRA_SIGDATA", "\n".join("%s: %s" % (k, deps[k]) for k in sorted(deps.keys())))
658}
659SSTATECREATEFUNCS += "target_add_sysroot_deps"
660