blob: 8372a4574ace27f3568aa7884667f6d05db00512 [file] [log] [blame]
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001# These directories will be staged in the sysroot
2SYSROOT_DIRS = " \
3 ${includedir} \
4 ${libdir} \
5 ${base_libdir} \
6 ${nonarch_base_libdir} \
7 ${datadir} \
Andrew Geisslerd1e89492021-02-12 15:35:20 -06008 /sysroot-only \
Patrick Williamsc0f7c042017-02-23 20:41:17 -06009"
10
11# These directories are also staged in the sysroot when they contain files that
12# are usable on the build system
13SYSROOT_DIRS_NATIVE = " \
14 ${bindir} \
15 ${sbindir} \
16 ${base_bindir} \
17 ${base_sbindir} \
18 ${libexecdir} \
19 ${sysconfdir} \
20 ${localstatedir} \
21"
Patrick Williams213cb262021-08-07 19:21:33 -050022SYSROOT_DIRS:append:class-native = " ${SYSROOT_DIRS_NATIVE}"
23SYSROOT_DIRS:append:class-cross = " ${SYSROOT_DIRS_NATIVE}"
24SYSROOT_DIRS:append:class-crosssdk = " ${SYSROOT_DIRS_NATIVE}"
Patrick Williamsc0f7c042017-02-23 20:41:17 -060025
26# These directories will not be staged in the sysroot
Andrew Geissler7e0e3c02022-02-25 20:34:39 +000027SYSROOT_DIRS_IGNORE = " \
Patrick Williamsc0f7c042017-02-23 20:41:17 -060028 ${mandir} \
29 ${docdir} \
30 ${infodir} \
Andrew Geisslerd1e89492021-02-12 15:35:20 -060031 ${datadir}/X11/locale \
Patrick Williamsc0f7c042017-02-23 20:41:17 -060032 ${datadir}/applications \
Andrew Geisslerd1e89492021-02-12 15:35:20 -060033 ${datadir}/bash-completion \
Patrick Williamsc0f7c042017-02-23 20:41:17 -060034 ${datadir}/fonts \
Brad Bishopc342db32019-05-15 21:57:59 -040035 ${datadir}/gtk-doc/html \
Andrew Geisslerd1e89492021-02-12 15:35:20 -060036 ${datadir}/installed-tests \
Brad Bishopc342db32019-05-15 21:57:59 -040037 ${datadir}/locale \
Patrick Williamsc0f7c042017-02-23 20:41:17 -060038 ${datadir}/pixmaps \
Andrew Geisslerd1e89492021-02-12 15:35:20 -060039 ${datadir}/terminfo \
Andrew Geissler82c905d2020-04-13 13:39:40 -050040 ${libdir}/${BPN}/ptest \
Patrick Williamsc0f7c042017-02-23 20:41:17 -060041"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050042
43sysroot_stage_dir() {
44 src="$1"
45 dest="$2"
46 # if the src doesn't exist don't do anything
47 if [ ! -d "$src" ]; then
48 return
49 fi
50
51 mkdir -p "$dest"
Andrew Geissler7e0e3c02022-02-25 20:34:39 +000052 rdest=$(realpath --relative-to="$src" "$dest")
Patrick Williamsc124f4f2015-09-15 14:41:29 -050053 (
54 cd $src
Andrew Geissler7e0e3c02022-02-25 20:34:39 +000055 find . -print0 | cpio --null -pdlu $rdest
Patrick Williamsc124f4f2015-09-15 14:41:29 -050056 )
57}
58
Patrick Williamsc124f4f2015-09-15 14:41:29 -050059sysroot_stage_dirs() {
60 from="$1"
61 to="$2"
62
Patrick Williamsc0f7c042017-02-23 20:41:17 -060063 for dir in ${SYSROOT_DIRS}; do
64 sysroot_stage_dir "$from$dir" "$to$dir"
65 done
66
67 # Remove directories we do not care about
Andrew Geissler7e0e3c02022-02-25 20:34:39 +000068 for dir in ${SYSROOT_DIRS_IGNORE}; do
Patrick Williamsc0f7c042017-02-23 20:41:17 -060069 rm -rf "$to$dir"
70 done
Patrick Williamsc124f4f2015-09-15 14:41:29 -050071}
72
73sysroot_stage_all() {
74 sysroot_stage_dirs ${D} ${SYSROOT_DESTDIR}
75}
76
77python sysroot_strip () {
Brad Bishopd7bf8c12018-02-25 22:55:05 -050078 inhibit_sysroot = d.getVar('INHIBIT_SYSROOT_STRIP')
79 if inhibit_sysroot and oe.types.boolean(inhibit_sysroot):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080080 return
Patrick Williamsc124f4f2015-09-15 14:41:29 -050081
Brad Bishopd7bf8c12018-02-25 22:55:05 -050082 dstdir = d.getVar('SYSROOT_DESTDIR')
Brad Bishop6e60e8b2018-02-01 10:27:11 -050083 pn = d.getVar('PN')
Andrew Geissler82c905d2020-04-13 13:39:40 -050084 libdir = d.getVar("libdir")
85 base_libdir = d.getVar("base_libdir")
Patrick Williams213cb262021-08-07 19:21:33 -050086 qa_already_stripped = 'already-stripped' in (d.getVar('INSANE_SKIP:' + pn) or "").split()
Brad Bishopd7bf8c12018-02-25 22:55:05 -050087 strip_cmd = d.getVar("STRIP")
Patrick Williamsc124f4f2015-09-15 14:41:29 -050088
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080089 oe.package.strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, d,
Brad Bishopd7bf8c12018-02-25 22:55:05 -050090 qa_already_stripped=qa_already_stripped)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050091}
92
93do_populate_sysroot[dirs] = "${SYSROOT_DESTDIR}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050094
95addtask populate_sysroot after do_install
96
97SYSROOT_PREPROCESS_FUNCS ?= ""
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050098SYSROOT_DESTDIR = "${WORKDIR}/sysroot-destdir"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050099
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500100python do_populate_sysroot () {
Andrew Geissler4b740dc2020-05-05 08:54:39 -0500101 # SYSROOT 'version' 2
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500102 bb.build.exec_func("sysroot_stage_all", d)
103 bb.build.exec_func("sysroot_strip", d)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500104 for f in (d.getVar('SYSROOT_PREPROCESS_FUNCS') or '').split():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500105 bb.build.exec_func(f, d)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500106 pn = d.getVar("PN")
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000107 multiprov = d.getVar("BB_MULTI_PROVIDER_ALLOWED").split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500108 provdir = d.expand("${SYSROOT_DESTDIR}${base_prefix}/sysroot-providers/")
109 bb.utils.mkdirhier(provdir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500110 for p in d.getVar("PROVIDES").split():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500111 if p in multiprov:
112 continue
113 p = p.replace("/", "_")
114 with open(provdir + p, "w") as f:
115 f.write(pn)
116}
117
118do_populate_sysroot[vardeps] += "${SYSROOT_PREPROCESS_FUNCS}"
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000119do_populate_sysroot[vardepsexclude] += "BB_MULTI_PROVIDER_ALLOWED"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500120
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500121POPULATESYSROOTDEPS = ""
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000122POPULATESYSROOTDEPS:class-target = "virtual/${MLPREFIX}${HOST_PREFIX}binutils:do_populate_sysroot"
123POPULATESYSROOTDEPS:class-nativesdk = "virtual/${HOST_PREFIX}binutils-crosssdk:do_populate_sysroot"
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500124do_populate_sysroot[depends] += "${POPULATESYSROOTDEPS}"
125
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500126SSTATETASKS += "do_populate_sysroot"
127do_populate_sysroot[cleandirs] = "${SYSROOT_DESTDIR}"
128do_populate_sysroot[sstate-inputdirs] = "${SYSROOT_DESTDIR}"
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500129do_populate_sysroot[sstate-outputdirs] = "${COMPONENTS_DIR}/${PACKAGE_ARCH}/${PN}"
130do_populate_sysroot[sstate-fixmedir] = "${COMPONENTS_DIR}/${PACKAGE_ARCH}/${PN}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500131
132python do_populate_sysroot_setscene () {
133 sstate_setscene(d)
134}
135addtask do_populate_sysroot_setscene
136
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500137def staging_copyfile(c, target, dest, postinsts, seendirs):
138 import errno
139
140 destdir = os.path.dirname(dest)
141 if destdir not in seendirs:
142 bb.utils.mkdirhier(destdir)
143 seendirs.add(destdir)
144 if "/usr/bin/postinst-" in c:
145 postinsts.append(dest)
146 if os.path.islink(c):
147 linkto = os.readlink(c)
148 if os.path.lexists(dest):
149 if not os.path.islink(dest):
150 raise OSError(errno.EEXIST, "Link %s already exists as a file" % dest, dest)
151 if os.readlink(dest) == linkto:
152 return dest
153 raise OSError(errno.EEXIST, "Link %s already exists to a different location? (%s vs %s)" % (dest, os.readlink(dest), linkto), dest)
154 os.symlink(linkto, dest)
155 #bb.warn(c)
156 else:
157 try:
158 os.link(c, dest)
159 except OSError as err:
160 if err.errno == errno.EXDEV:
161 bb.utils.copyfile(c, dest)
162 else:
163 raise
164 return dest
165
166def staging_copydir(c, target, dest, seendirs):
167 if dest not in seendirs:
168 bb.utils.mkdirhier(dest)
169 seendirs.add(dest)
170
171def staging_processfixme(fixme, target, recipesysroot, recipesysrootnative, d):
172 import subprocess
173
174 if not fixme:
175 return
176 cmd = "sed -e 's:^[^/]*/:%s/:g' %s | xargs sed -i -e 's:FIXMESTAGINGDIRTARGET:%s:g; s:FIXMESTAGINGDIRHOST:%s:g'" % (target, " ".join(fixme), recipesysroot, recipesysrootnative)
Brad Bishop15ae2502019-06-18 21:44:24 -0400177 for fixmevar in ['PSEUDO_SYSROOT', 'HOSTTOOLS_DIR', 'PKGDATA_DIR', 'PSEUDO_LOCALSTATEDIR', 'LOGFIFO']:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500178 fixme_path = d.getVar(fixmevar)
179 cmd += " -e 's:FIXME_%s:%s:g'" % (fixmevar, fixme_path)
180 bb.debug(2, cmd)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400181 subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500182
183
184def staging_populate_sysroot_dir(targetsysroot, nativesysroot, native, d):
185 import glob
186 import subprocess
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500187 import errno
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500188
189 fixme = []
190 postinsts = []
191 seendirs = set()
192 stagingdir = d.getVar("STAGING_DIR")
193 if native:
194 pkgarchs = ['${BUILD_ARCH}', '${BUILD_ARCH}_*']
195 targetdir = nativesysroot
196 else:
197 pkgarchs = ['${MACHINE_ARCH}']
198 pkgarchs = pkgarchs + list(reversed(d.getVar("PACKAGE_EXTRA_ARCHS").split()))
199 pkgarchs.append('allarch')
200 targetdir = targetsysroot
201
202 bb.utils.mkdirhier(targetdir)
203 for pkgarch in pkgarchs:
204 for manifest in glob.glob(d.expand("${SSTATE_MANIFESTS}/manifest-%s-*.populate_sysroot" % pkgarch)):
205 if manifest.endswith("-initial.populate_sysroot"):
Brad Bishop79641f22019-09-10 07:20:22 -0400206 # skip libgcc-initial due to file overlap
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500207 continue
Brad Bishop19323692019-04-05 15:28:33 -0400208 if not native and (manifest.endswith("-native.populate_sysroot") or "nativesdk-" in manifest):
209 continue
210 if native and not (manifest.endswith("-native.populate_sysroot") or manifest.endswith("-cross.populate_sysroot") or "-cross-" in manifest):
211 continue
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500212 tmanifest = targetdir + "/" + os.path.basename(manifest)
213 if os.path.exists(tmanifest):
214 continue
215 try:
216 os.link(manifest, tmanifest)
217 except OSError as err:
218 if err.errno == errno.EXDEV:
219 bb.utils.copyfile(manifest, tmanifest)
220 else:
221 raise
222 with open(manifest, "r") as f:
223 for l in f:
224 l = l.strip()
225 if l.endswith("/fixmepath"):
226 fixme.append(l)
227 continue
228 if l.endswith("/fixmepath.cmd"):
229 continue
230 dest = l.replace(stagingdir, "")
231 dest = targetdir + "/" + "/".join(dest.split("/")[3:])
232 if l.endswith("/"):
233 staging_copydir(l, targetdir, dest, seendirs)
234 continue
235 try:
236 staging_copyfile(l, targetdir, dest, postinsts, seendirs)
237 except FileExistsError:
238 continue
239
240 staging_processfixme(fixme, targetdir, targetsysroot, nativesysroot, d)
241 for p in postinsts:
Brad Bishop316dfdd2018-06-25 12:45:53 -0400242 subprocess.check_output(p, shell=True, stderr=subprocess.STDOUT)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500243
244#
245# Manifests here are complicated. The main sysroot area has the unpacked sstate
246# which us unrelocated and tracked by the main sstate manifests. Each recipe
247# specific sysroot has manifests for each dependency that is installed there.
248# The task hash is used to tell whether the data needs to be reinstalled. We
249# use a symlink to point to the currently installed hash. There is also a
250# "complete" stamp file which is used to mark if installation completed. If
251# something fails (e.g. a postinst), this won't get written and we would
252# remove and reinstall the dependency. This also means partially installed
253# dependencies should get cleaned up correctly.
254#
255
256python extend_recipe_sysroot() {
257 import copy
258 import subprocess
259 import errno
260 import collections
261 import glob
262
263 taskdepdata = d.getVar("BB_TASKDEPDATA", False)
264 mytaskname = d.getVar("BB_RUNTASK")
265 if mytaskname.endswith("_setscene"):
266 mytaskname = mytaskname.replace("_setscene", "")
267 workdir = d.getVar("WORKDIR")
268 #bb.warn(str(taskdepdata))
269 pn = d.getVar("PN")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500270 stagingdir = d.getVar("STAGING_DIR")
271 sharedmanifests = d.getVar("COMPONENTS_DIR") + "/manifests"
272 recipesysroot = d.getVar("RECIPE_SYSROOT")
273 recipesysrootnative = d.getVar("RECIPE_SYSROOT_NATIVE")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500274
275 # Detect bitbake -b usage
276 nodeps = d.getVar("BB_LIMITEDDEPS") or False
277 if nodeps:
278 lock = bb.utils.lockfile(recipesysroot + "/sysroot.lock")
279 staging_populate_sysroot_dir(recipesysroot, recipesysrootnative, True, d)
280 staging_populate_sysroot_dir(recipesysroot, recipesysrootnative, False, d)
281 bb.utils.unlockfile(lock)
282 return
283
284 start = None
285 configuredeps = []
Andrew Geissler82c905d2020-04-13 13:39:40 -0500286 owntaskdeps = []
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500287 for dep in taskdepdata:
288 data = taskdepdata[dep]
289 if data[1] == mytaskname and data[0] == pn:
290 start = dep
Andrew Geissler82c905d2020-04-13 13:39:40 -0500291 elif data[0] == pn:
292 owntaskdeps.append(data[1])
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500293 if start is None:
294 bb.fatal("Couldn't find ourself in BB_TASKDEPDATA?")
295
296 # We need to figure out which sysroot files we need to expose to this task.
297 # This needs to match what would get restored from sstate, which is controlled
298 # ultimately by calls from bitbake to setscene_depvalid().
299 # That function expects a setscene dependency tree. We build a dependency tree
300 # condensed to inter-sstate task dependencies, similar to that used by setscene
301 # tasks. We can then call into setscene_depvalid() and decide
302 # which dependencies we can "see" and should expose in the recipe specific sysroot.
303 setscenedeps = copy.deepcopy(taskdepdata)
304
305 start = set([start])
306
307 sstatetasks = d.getVar("SSTATETASKS").split()
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800308 # Add recipe specific tasks referenced by setscene_depvalid()
309 sstatetasks.append("do_stash_locale")
Andrew Geissler5199d832021-09-24 16:47:35 -0500310 sstatetasks.append("do_deploy")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500311
312 def print_dep_tree(deptree):
313 data = ""
314 for dep in deptree:
315 deps = " " + "\n ".join(deptree[dep][3]) + "\n"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800316 data = data + "%s:\n %s\n %s\n%s %s\n %s\n" % (deptree[dep][0], deptree[dep][1], deptree[dep][2], deps, deptree[dep][4], deptree[dep][5])
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500317 return data
318
319 #bb.note("Full dep tree is:\n%s" % print_dep_tree(taskdepdata))
320
321 #bb.note(" start2 is %s" % str(start))
322
323 # If start is an sstate task (like do_package) we need to add in its direct dependencies
324 # else the code below won't recurse into them.
325 for dep in set(start):
326 for dep2 in setscenedeps[dep][3]:
327 start.add(dep2)
328 start.remove(dep)
329
330 #bb.note(" start3 is %s" % str(start))
331
332 # Create collapsed do_populate_sysroot -> do_populate_sysroot tree
333 for dep in taskdepdata:
334 data = setscenedeps[dep]
335 if data[1] not in sstatetasks:
336 for dep2 in setscenedeps:
337 data2 = setscenedeps[dep2]
338 if dep in data2[3]:
339 data2[3].update(setscenedeps[dep][3])
340 data2[3].remove(dep)
341 if dep in start:
342 start.update(setscenedeps[dep][3])
343 start.remove(dep)
344 del setscenedeps[dep]
345
346 # Remove circular references
347 for dep in setscenedeps:
348 if dep in setscenedeps[dep][3]:
349 setscenedeps[dep][3].remove(dep)
350
351 #bb.note("Computed dep tree is:\n%s" % print_dep_tree(setscenedeps))
352 #bb.note(" start is %s" % str(start))
353
354 # Direct dependencies should be present and can be depended upon
355 for dep in set(start):
356 if setscenedeps[dep][1] == "do_populate_sysroot":
357 if dep not in configuredeps:
358 configuredeps.append(dep)
359 bb.note("Direct dependencies are %s" % str(configuredeps))
360 #bb.note(" or %s" % str(start))
361
362 msgbuf = []
363 # Call into setscene_depvalid for each sub-dependency and only copy sysroot files
364 # for ones that would be restored from sstate.
365 done = list(start)
366 next = list(start)
367 while next:
368 new = []
369 for dep in next:
370 data = setscenedeps[dep]
371 for datadep in data[3]:
372 if datadep in done:
373 continue
374 taskdeps = {}
375 taskdeps[dep] = setscenedeps[dep][:2]
376 taskdeps[datadep] = setscenedeps[datadep][:2]
377 retval = setscene_depvalid(datadep, taskdeps, [], d, msgbuf)
378 if retval:
379 msgbuf.append("Skipping setscene dependency %s for installation into the sysroot" % datadep)
380 continue
381 done.append(datadep)
382 new.append(datadep)
383 if datadep not in configuredeps and setscenedeps[datadep][1] == "do_populate_sysroot":
384 configuredeps.append(datadep)
385 msgbuf.append("Adding dependency on %s" % setscenedeps[datadep][0])
386 else:
387 msgbuf.append("Following dependency on %s" % setscenedeps[datadep][0])
388 next = new
389
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500390 # This logging is too verbose for day to day use sadly
391 #bb.debug(2, "\n".join(msgbuf))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500392
393 depdir = recipesysrootnative + "/installeddeps"
394 bb.utils.mkdirhier(depdir)
395 bb.utils.mkdirhier(sharedmanifests)
396
397 lock = bb.utils.lockfile(recipesysroot + "/sysroot.lock")
398
399 fixme = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500400 seendirs = set()
401 postinsts = []
402 multilibs = {}
403 manifests = {}
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500404 # All files that we're going to be installing, to find conflicts.
405 fileset = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500406
Andrew Geisslerd5838332022-05-27 11:33:10 -0500407 invalidate_tasks = set()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500408 for f in os.listdir(depdir):
Andrew Geisslerd5838332022-05-27 11:33:10 -0500409 removed = []
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500410 if not f.endswith(".complete"):
411 continue
412 f = depdir + "/" + f
413 if os.path.islink(f) and not os.path.exists(f):
414 bb.note("%s no longer exists, removing from sysroot" % f)
415 lnk = os.readlink(f.replace(".complete", ""))
Andrew Geissler5f350902021-07-23 13:09:54 -0400416 sstate_clean_manifest(depdir + "/" + lnk, d, canrace=True, prefix=workdir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500417 os.unlink(f)
418 os.unlink(f.replace(".complete", ""))
Andrew Geisslerd5838332022-05-27 11:33:10 -0500419 removed.append(os.path.basename(f.replace(".complete", "")))
420
421 # If we've removed files from the sysroot above, the task that installed them may still
422 # have a stamp file present for the task. This is probably invalid right now but may become
423 # valid again if the user were to change configuration back for example. Since we've removed
424 # the files a task might need, remove the stamp file too to force it to rerun.
425 # YOCTO #14790
426 if removed:
427 for i in glob.glob(depdir + "/index.*"):
428 if i.endswith("." + mytaskname):
429 continue
430 with open(i, "r") as f:
431 for l in f:
432 if l.startswith("TaskDeps:"):
433 continue
434 l = l.strip()
435 if l in removed:
436 invalidate_tasks.add(i.rsplit(".", 1)[1])
437 break
438 for t in invalidate_tasks:
439 bb.note("Invalidating stamps for task %s" % t)
440 bb.build.clean_stamp(t, d)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500441
442 installed = []
443 for dep in configuredeps:
444 c = setscenedeps[dep][0]
445 if mytaskname in ["do_sdk_depends", "do_populate_sdk_ext"] and c.endswith("-initial"):
446 bb.note("Skipping initial setscene dependency %s for installation into the sysroot" % c)
447 continue
448 installed.append(c)
449
450 # We want to remove anything which this task previously installed but is no longer a dependency
451 taskindex = depdir + "/" + "index." + mytaskname
452 if os.path.exists(taskindex):
453 potential = []
454 with open(taskindex, "r") as f:
455 for l in f:
456 l = l.strip()
457 if l not in installed:
458 fl = depdir + "/" + l
459 if not os.path.exists(fl):
460 # Was likely already uninstalled
461 continue
462 potential.append(l)
Andrew Geissler82c905d2020-04-13 13:39:40 -0500463 # We need to ensure no other task needs this dependency. We hold the sysroot
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500464 # lock so we ca search the indexes to check
465 if potential:
466 for i in glob.glob(depdir + "/index.*"):
467 if i.endswith("." + mytaskname):
468 continue
469 with open(i, "r") as f:
470 for l in f:
Andrew Geissler82c905d2020-04-13 13:39:40 -0500471 if l.startswith("TaskDeps:"):
472 prevtasks = l.split()[1:]
473 if mytaskname in prevtasks:
474 # We're a dependency of this task so we can clear items out the sysroot
475 break
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500476 l = l.strip()
477 if l in potential:
478 potential.remove(l)
479 for l in potential:
480 fl = depdir + "/" + l
481 bb.note("Task %s no longer depends on %s, removing from sysroot" % (mytaskname, l))
482 lnk = os.readlink(fl)
Andrew Geissler5f350902021-07-23 13:09:54 -0400483 sstate_clean_manifest(depdir + "/" + lnk, d, canrace=True, prefix=workdir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500484 os.unlink(fl)
485 os.unlink(fl + ".complete")
486
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500487 msg_exists = []
488 msg_adding = []
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800489
Brad Bishop1d80a2e2019-11-15 16:35:03 -0500490 # Handle all removals first since files may move between recipes
491 for dep in configuredeps:
492 c = setscenedeps[dep][0]
493 if c not in installed:
494 continue
495 taskhash = setscenedeps[dep][5]
496 taskmanifest = depdir + "/" + c + "." + taskhash
497
498 if os.path.exists(depdir + "/" + c):
499 lnk = os.readlink(depdir + "/" + c)
500 if lnk == c + "." + taskhash and os.path.exists(depdir + "/" + c + ".complete"):
501 continue
502 else:
503 bb.note("%s exists in sysroot, but is stale (%s vs. %s), removing." % (c, lnk, c + "." + taskhash))
Andrew Geissler5f350902021-07-23 13:09:54 -0400504 sstate_clean_manifest(depdir + "/" + lnk, d, canrace=True, prefix=workdir)
Brad Bishop1d80a2e2019-11-15 16:35:03 -0500505 os.unlink(depdir + "/" + c)
506 if os.path.lexists(depdir + "/" + c + ".complete"):
507 os.unlink(depdir + "/" + c + ".complete")
508 elif os.path.lexists(depdir + "/" + c):
509 os.unlink(depdir + "/" + c)
510
Andrew Geissler82c905d2020-04-13 13:39:40 -0500511 binfiles = {}
Brad Bishop1d80a2e2019-11-15 16:35:03 -0500512 # Now handle installs
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500513 for dep in configuredeps:
514 c = setscenedeps[dep][0]
515 if c not in installed:
516 continue
517 taskhash = setscenedeps[dep][5]
518 taskmanifest = depdir + "/" + c + "." + taskhash
519
520 if os.path.exists(depdir + "/" + c):
521 lnk = os.readlink(depdir + "/" + c)
522 if lnk == c + "." + taskhash and os.path.exists(depdir + "/" + c + ".complete"):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500523 msg_exists.append(c)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500524 continue
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500525
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500526 msg_adding.append(c)
527
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500528 os.symlink(c + "." + taskhash, depdir + "/" + c)
529
Brad Bishop316dfdd2018-06-25 12:45:53 -0400530 manifest, d2 = oe.sstatesig.find_sstate_manifest(c, setscenedeps[dep][2], "populate_sysroot", d, multilibs)
Brad Bishopd5ae7d92018-06-14 09:52:03 -0700531 if d2 is not d:
532 # If we don't do this, the recipe sysroot will be placed in the wrong WORKDIR for multilibs
533 # We need a consistent WORKDIR for the image
534 d2.setVar("WORKDIR", d.getVar("WORKDIR"))
Brad Bishop316dfdd2018-06-25 12:45:53 -0400535 destsysroot = d2.getVar("RECIPE_SYSROOT")
Brad Bishopd5ae7d92018-06-14 09:52:03 -0700536 # We put allarch recipes into the default sysroot
537 if manifest and "allarch" in manifest:
538 destsysroot = d.getVar("RECIPE_SYSROOT")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500539
540 native = False
Brad Bishop316dfdd2018-06-25 12:45:53 -0400541 if c.endswith("-native") or "-cross-" in c or "-crosssdk" in c:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500542 native = True
Brad Bishop316dfdd2018-06-25 12:45:53 -0400543
544 if manifest:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500545 newmanifest = collections.OrderedDict()
Brad Bishopd5ae7d92018-06-14 09:52:03 -0700546 targetdir = destsysroot
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500547 if native:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500548 targetdir = recipesysrootnative
Brad Bishopd5ae7d92018-06-14 09:52:03 -0700549 if targetdir not in fixme:
550 fixme[targetdir] = []
551 fm = fixme[targetdir]
552
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500553 with open(manifest, "r") as f:
554 manifests[dep] = manifest
555 for l in f:
556 l = l.strip()
557 if l.endswith("/fixmepath"):
558 fm.append(l)
559 continue
560 if l.endswith("/fixmepath.cmd"):
561 continue
562 dest = l.replace(stagingdir, "")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500563 dest = "/" + "/".join(dest.split("/")[3:])
564 newmanifest[l] = targetdir + dest
565
566 # Check if files have already been installed by another
567 # recipe and abort if they have, explaining what recipes are
568 # conflicting.
569 hashname = targetdir + dest
570 if not hashname.endswith("/"):
571 if hashname in fileset:
572 bb.fatal("The file %s is installed by both %s and %s, aborting" % (dest, c, fileset[hashname]))
573 else:
574 fileset[hashname] = c
575
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500576 # Having multiple identical manifests in each sysroot eats diskspace so
577 # create a shared pool of them and hardlink if we can.
578 # We create the manifest in advance so that if something fails during installation,
579 # or the build is interrupted, subsequent exeuction can cleanup.
580 sharedm = sharedmanifests + "/" + os.path.basename(taskmanifest)
581 if not os.path.exists(sharedm):
582 smlock = bb.utils.lockfile(sharedm + ".lock")
583 # Can race here. You'd think it just means we may not end up with all copies hardlinked to each other
584 # but python can lose file handles so we need to do this under a lock.
585 if not os.path.exists(sharedm):
586 with open(sharedm, 'w') as m:
587 for l in newmanifest:
588 dest = newmanifest[l]
589 m.write(dest.replace(workdir + "/", "") + "\n")
590 bb.utils.unlockfile(smlock)
591 try:
592 os.link(sharedm, taskmanifest)
593 except OSError as err:
594 if err.errno == errno.EXDEV:
595 bb.utils.copyfile(sharedm, taskmanifest)
596 else:
597 raise
598 # Finally actually install the files
599 for l in newmanifest:
600 dest = newmanifest[l]
601 if l.endswith("/"):
602 staging_copydir(l, targetdir, dest, seendirs)
603 continue
Andrew Geissler82c905d2020-04-13 13:39:40 -0500604 if "/bin/" in l or "/sbin/" in l:
605 # defer /*bin/* files until last in case they need libs
606 binfiles[l] = (targetdir, dest)
607 else:
608 staging_copyfile(l, targetdir, dest, postinsts, seendirs)
609
610 # Handle deferred binfiles
611 for l in binfiles:
612 (targetdir, dest) = binfiles[l]
613 staging_copyfile(l, targetdir, dest, postinsts, seendirs)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500614
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500615 bb.note("Installed into sysroot: %s" % str(msg_adding))
616 bb.note("Skipping as already exists in sysroot: %s" % str(msg_exists))
617
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500618 for f in fixme:
Brad Bishopd5ae7d92018-06-14 09:52:03 -0700619 staging_processfixme(fixme[f], f, recipesysroot, recipesysrootnative, d)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500620
621 for p in postinsts:
Brad Bishop316dfdd2018-06-25 12:45:53 -0400622 subprocess.check_output(p, shell=True, stderr=subprocess.STDOUT)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500623
624 for dep in manifests:
625 c = setscenedeps[dep][0]
626 os.symlink(manifests[dep], depdir + "/" + c + ".complete")
627
628 with open(taskindex, "w") as f:
Andrew Geissler82c905d2020-04-13 13:39:40 -0500629 f.write("TaskDeps: " + " ".join(owntaskdeps) + "\n")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500630 for l in sorted(installed):
631 f.write(l + "\n")
632
633 bb.utils.unlockfile(lock)
634}
635extend_recipe_sysroot[vardepsexclude] += "MACHINE_ARCH PACKAGE_EXTRA_ARCHS SDK_ARCH BUILD_ARCH SDK_OS BB_TASKDEPDATA"
636
Brad Bishop19323692019-04-05 15:28:33 -0400637do_prepare_recipe_sysroot[deptask] = "do_populate_sysroot"
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500638python do_prepare_recipe_sysroot () {
639 bb.build.exec_func("extend_recipe_sysroot", d)
640}
641addtask do_prepare_recipe_sysroot before do_configure after do_fetch
642
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500643python staging_taskhandler() {
644 bbtasks = e.tasklist
645 for task in bbtasks:
646 deps = d.getVarFlag(task, "depends")
Andrew Geisslerc3d88e42020-10-02 09:45:00 -0500647 if task == "do_configure" or (deps and "populate_sysroot" in deps):
Andrew Geisslereff27472021-10-29 15:35:00 -0500648 d.prependVarFlag(task, "prefuncs", "extend_recipe_sysroot ")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500649}
650staging_taskhandler[eventmask] = "bb.event.RecipeTaskPreProcess"
651addhandler staging_taskhandler
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000652
653
654#
655# Target build output, stored in do_populate_sysroot or do_package can depend
656# not only upon direct dependencies but also indirect ones. A good example is
657# linux-libc-headers. The toolchain depends on this but most target recipes do
658# not. There are some headers which are not used by the toolchain build and do
659# not change the toolchain task output, hence the task hashes can change without
660# changing the sysroot output of that recipe yet they can influence others.
661#
662# A specific example is rtc.h which can change rtcwake.c in util-linux but is not
663# used in the glibc or gcc build. To account for this, we need to account for the
664# populate_sysroot hashes in the task output hashes.
665#
666python target_add_sysroot_deps () {
667 current_task = "do_" + d.getVar("BB_CURRENTTASK")
668 if current_task not in ["do_populate_sysroot", "do_package"]:
669 return
670
671 pn = d.getVar("PN")
672 if pn.endswith("-native"):
673 return
674
675 taskdepdata = d.getVar("BB_TASKDEPDATA", False)
676 deps = {}
677 for dep in taskdepdata.values():
Patrick Williams03907ee2022-05-01 06:28:52 -0500678 if dep[1] == "do_populate_sysroot" and not dep[0].endswith(("-native", "-initial")) and "-cross-" not in dep[0] and dep[0] != pn:
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000679 deps[dep[0]] = dep[6]
680
681 d.setVar("HASHEQUIV_EXTRA_SIGDATA", "\n".join("%s: %s" % (k, deps[k]) for k in sorted(deps.keys())))
682}
683SSTATECREATEFUNCS += "target_add_sysroot_deps"
684