blob: daa96f3b63bb377b9cdc626c56be71d7d668f71b [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001#
2# Records history of build output in order to detect regressions
3#
4# Based in part on testlab.bbclass and packagehistory.bbclass
5#
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05006# Copyright (C) 2011-2016 Intel Corporation
Patrick Williamsc124f4f2015-09-15 14:41:29 -05007# Copyright (C) 2007-2011 Koen Kooi <koen@openembedded.org>
8#
9
Andrew Geisslerc9f78652020-09-18 14:11:35 -050010inherit image-artifact-names
11
Patrick Williamsc124f4f2015-09-15 14:41:29 -050012BUILDHISTORY_FEATURES ?= "image package sdk"
13BUILDHISTORY_DIR ?= "${TOPDIR}/buildhistory"
14BUILDHISTORY_DIR_IMAGE = "${BUILDHISTORY_DIR}/images/${MACHINE_ARCH}/${TCLIBC}/${IMAGE_BASENAME}"
15BUILDHISTORY_DIR_PACKAGE = "${BUILDHISTORY_DIR}/packages/${MULTIMACH_TARGET_SYS}/${PN}"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050016
17# Setting this to non-empty will remove the old content of the buildhistory as part of
18# the current bitbake invocation and replace it with information about what was built
19# during the build.
20#
21# This is meant to be used in continuous integration (CI) systems when invoking bitbake
22# for full world builds. The effect in that case is that information about packages
23# that no longer get build also gets removed from the buildhistory, which is not
24# the case otherwise.
25#
26# The advantage over manually cleaning the buildhistory outside of bitbake is that
27# the "version-going-backwards" check still works. When relying on that, be careful
28# about failed world builds: they will lead to incomplete information in the
29# buildhistory because information about packages that could not be built will
30# also get removed. A CI system should handle that by discarding the buildhistory
31# of failed builds.
32#
33# The expected usage is via auto.conf, but passing via the command line also works
34# with: BB_ENV_EXTRAWHITE=BUILDHISTORY_RESET BUILDHISTORY_RESET=1
35BUILDHISTORY_RESET ?= ""
36
37BUILDHISTORY_OLD_DIR = "${BUILDHISTORY_DIR}/${@ "old" if "${BUILDHISTORY_RESET}" else ""}"
38BUILDHISTORY_OLD_DIR_PACKAGE = "${BUILDHISTORY_OLD_DIR}/packages/${MULTIMACH_TARGET_SYS}/${PN}"
39BUILDHISTORY_DIR_SDK = "${BUILDHISTORY_DIR}/sdk/${SDK_NAME}${SDK_EXT}/${IMAGE_BASENAME}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050040BUILDHISTORY_IMAGE_FILES ?= "/etc/passwd /etc/group"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050041BUILDHISTORY_SDK_FILES ?= "conf/local.conf conf/bblayers.conf conf/auto.conf conf/locked-sigs.inc conf/devtool.conf"
Brad Bishop316dfdd2018-06-25 12:45:53 -040042BUILDHISTORY_COMMIT ?= "1"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050043BUILDHISTORY_COMMIT_AUTHOR ?= "buildhistory <buildhistory@${DISTRO}>"
44BUILDHISTORY_PUSH_REPO ?= ""
Andrew Geissler82c905d2020-04-13 13:39:40 -050045BUILDHISTORY_TAG ?= "build"
Andrew Geissler09036742021-06-25 14:25:14 -050046BUILDHISTORY_PATH_PREFIX_STRIP ?= ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -050047
Patrick Williams213cb262021-08-07 19:21:33 -050048SSTATEPOSTINSTFUNCS:append = " buildhistory_emit_pkghistory"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050049# We want to avoid influencing the signatures of sstate tasks - first the function itself:
Patrick Williamsc124f4f2015-09-15 14:41:29 -050050sstate_install[vardepsexclude] += "buildhistory_emit_pkghistory"
51# then the value added to SSTATEPOSTINSTFUNCS:
52SSTATEPOSTINSTFUNCS[vardepvalueexclude] .= "| buildhistory_emit_pkghistory"
53
Brad Bishop6e60e8b2018-02-01 10:27:11 -050054# Similarly for our function that gets the output signatures
Patrick Williams213cb262021-08-07 19:21:33 -050055SSTATEPOSTUNPACKFUNCS:append = " buildhistory_emit_outputsigs"
Brad Bishop6e60e8b2018-02-01 10:27:11 -050056sstate_installpkgdir[vardepsexclude] += "buildhistory_emit_outputsigs"
57SSTATEPOSTUNPACKFUNCS[vardepvalueexclude] .= "| buildhistory_emit_outputsigs"
58
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050059# All items excepts those listed here will be removed from a recipe's
60# build history directory by buildhistory_emit_pkghistory(). This is
61# necessary because some of these items (package directories, files that
62# we no longer emit) might be obsolete.
Patrick Williamsc124f4f2015-09-15 14:41:29 -050063#
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050064# When extending build history, derive your class from buildhistory.bbclass
65# and extend this list here with the additional files created by the derived
66# class.
Brad Bishop96ff1982019-08-19 13:50:42 -040067BUILDHISTORY_PRESERVE = "latest latest_srcrev sysroot"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050068
Patrick Williamsc0f7c042017-02-23 20:41:17 -060069PATCH_GIT_USER_EMAIL ?= "buildhistory@oe"
70PATCH_GIT_USER_NAME ?= "OpenEmbedded"
71
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050072#
Brad Bishop96ff1982019-08-19 13:50:42 -040073# Write out the contents of the sysroot
74#
75buildhistory_emit_sysroot() {
76 mkdir --parents ${BUILDHISTORY_DIR_PACKAGE}
77 case ${CLASSOVERRIDE} in
78 class-native|class-cross|class-crosssdk)
79 BASE=${SYSROOT_DESTDIR}/${STAGING_DIR_NATIVE}
80 ;;
81 *)
82 BASE=${SYSROOT_DESTDIR}
83 ;;
84 esac
85 buildhistory_list_files_no_owners $BASE ${BUILDHISTORY_DIR_PACKAGE}/sysroot
86}
87
88#
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050089# Write out metadata about this package for comparison when writing future packages
Patrick Williamsc124f4f2015-09-15 14:41:29 -050090#
91python buildhistory_emit_pkghistory() {
Brad Bishop96ff1982019-08-19 13:50:42 -040092 if d.getVar('BB_CURRENTTASK') in ['populate_sysroot', 'populate_sysroot_setscene']:
93 bb.build.exec_func("buildhistory_emit_sysroot", d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050094 return 0
95
Brad Bishop6e60e8b2018-02-01 10:27:11 -050096 if not "package" in (d.getVar('BUILDHISTORY_FEATURES') or "").split():
Patrick Williamsc124f4f2015-09-15 14:41:29 -050097 return 0
98
Andrew Geissler595f6302022-01-24 19:11:47 +000099 if d.getVar('BB_CURRENTTASK') in ['package', 'package_setscene']:
100 # Create files-in-<package-name>.txt files containing a list of files of each recipe's package
101 bb.build.exec_func("buildhistory_list_pkg_files", d)
102 return 0
103
104 if not d.getVar('BB_CURRENTTASK') in ['packagedata', 'packagedata_setscene']:
105 return 0
106
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500107 import re
108 import json
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800109 import shlex
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500110 import errno
111
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500112 pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE')
113 oldpkghistdir = d.getVar('BUILDHISTORY_OLD_DIR_PACKAGE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500114
115 class RecipeInfo:
116 def __init__(self, name):
117 self.name = name
118 self.pe = "0"
119 self.pv = "0"
120 self.pr = "r0"
121 self.depends = ""
122 self.packages = ""
123 self.srcrev = ""
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500124 self.layer = ""
Andrew Geissler6ce62a22020-11-30 19:58:47 -0600125 self.license = ""
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500126 self.config = ""
Andrew Geissler4c19ea12020-10-27 13:52:24 -0500127 self.src_uri = ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500128
129
130 class PackageInfo:
131 def __init__(self, name):
132 self.name = name
133 self.pe = "0"
134 self.pv = "0"
135 self.pr = "r0"
136 # pkg/pkge/pkgv/pkgr should be empty because we want to be able to default them
137 self.pkg = ""
138 self.pkge = ""
139 self.pkgv = ""
140 self.pkgr = ""
141 self.size = 0
142 self.depends = ""
143 self.rprovides = ""
144 self.rdepends = ""
145 self.rrecommends = ""
146 self.rsuggests = ""
147 self.rreplaces = ""
148 self.rconflicts = ""
149 self.files = ""
150 self.filelist = ""
151 # Variables that need to be written to their own separate file
152 self.filevars = dict.fromkeys(['pkg_preinst', 'pkg_postinst', 'pkg_prerm', 'pkg_postrm'])
153
154 # Should check PACKAGES here to see if anything removed
155
156 def readPackageInfo(pkg, histfile):
157 pkginfo = PackageInfo(pkg)
158 with open(histfile, "r") as f:
159 for line in f:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500160 lns = line.split('=', 1)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500161 name = lns[0].strip()
162 value = lns[1].strip(" \t\r\n").strip('"')
163 if name == "PE":
164 pkginfo.pe = value
165 elif name == "PV":
166 pkginfo.pv = value
167 elif name == "PR":
168 pkginfo.pr = value
169 elif name == "PKG":
170 pkginfo.pkg = value
171 elif name == "PKGE":
172 pkginfo.pkge = value
173 elif name == "PKGV":
174 pkginfo.pkgv = value
175 elif name == "PKGR":
176 pkginfo.pkgr = value
177 elif name == "RPROVIDES":
178 pkginfo.rprovides = value
179 elif name == "RDEPENDS":
180 pkginfo.rdepends = value
181 elif name == "RRECOMMENDS":
182 pkginfo.rrecommends = value
183 elif name == "RSUGGESTS":
184 pkginfo.rsuggests = value
185 elif name == "RREPLACES":
186 pkginfo.rreplaces = value
187 elif name == "RCONFLICTS":
188 pkginfo.rconflicts = value
189 elif name == "PKGSIZE":
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600190 pkginfo.size = int(value)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500191 elif name == "FILES":
192 pkginfo.files = value
193 elif name == "FILELIST":
194 pkginfo.filelist = value
195 # Apply defaults
196 if not pkginfo.pkg:
197 pkginfo.pkg = pkginfo.name
198 if not pkginfo.pkge:
199 pkginfo.pkge = pkginfo.pe
200 if not pkginfo.pkgv:
201 pkginfo.pkgv = pkginfo.pv
202 if not pkginfo.pkgr:
203 pkginfo.pkgr = pkginfo.pr
204 return pkginfo
205
206 def getlastpkgversion(pkg):
207 try:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500208 histfile = os.path.join(oldpkghistdir, pkg, "latest")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500209 return readPackageInfo(pkg, histfile)
210 except EnvironmentError:
211 return None
212
213 def sortpkglist(string):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500214 pkgiter = re.finditer(r'[a-zA-Z0-9.+-]+( \([><=]+[^)]+\))?', string, 0)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500215 pkglist = [p.group(0) for p in pkgiter]
216 pkglist.sort()
217 return ' '.join(pkglist)
218
219 def sortlist(string):
220 items = string.split(' ')
221 items.sort()
222 return ' '.join(items)
223
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500224 pn = d.getVar('PN')
225 pe = d.getVar('PE') or "0"
226 pv = d.getVar('PV')
227 pr = d.getVar('PR')
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500228 layer = bb.utils.get_file_layer(d.getVar('FILE'), d)
Andrew Geissler6ce62a22020-11-30 19:58:47 -0600229 license = d.getVar('LICENSE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500230
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500231 pkgdata_dir = d.getVar('PKGDATA_DIR')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500232 packages = ""
233 try:
234 with open(os.path.join(pkgdata_dir, pn)) as f:
235 for line in f.readlines():
236 if line.startswith('PACKAGES: '):
237 packages = oe.utils.squashspaces(line.split(': ', 1)[1])
238 break
239 except IOError as e:
240 if e.errno == errno.ENOENT:
241 # Probably a -cross recipe, just ignore
242 return 0
243 else:
244 raise
245
246 packagelist = packages.split()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500247 preserve = d.getVar('BUILDHISTORY_PRESERVE').split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500248 if not os.path.exists(pkghistdir):
249 bb.utils.mkdirhier(pkghistdir)
250 else:
251 # Remove files for packages that no longer exist
252 for item in os.listdir(pkghistdir):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500253 if item not in preserve:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500254 if item not in packagelist:
255 itempath = os.path.join(pkghistdir, item)
256 if os.path.isdir(itempath):
257 for subfile in os.listdir(itempath):
258 os.unlink(os.path.join(itempath, subfile))
259 os.rmdir(itempath)
260 else:
261 os.unlink(itempath)
262
263 rcpinfo = RecipeInfo(pn)
264 rcpinfo.pe = pe
265 rcpinfo.pv = pv
266 rcpinfo.pr = pr
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500267 rcpinfo.depends = sortlist(oe.utils.squashspaces(d.getVar('DEPENDS') or ""))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500268 rcpinfo.packages = packages
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500269 rcpinfo.layer = layer
Andrew Geissler6ce62a22020-11-30 19:58:47 -0600270 rcpinfo.license = license
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500271 rcpinfo.config = sortlist(oe.utils.squashspaces(d.getVar('PACKAGECONFIG') or ""))
Andrew Geissler4c19ea12020-10-27 13:52:24 -0500272 rcpinfo.src_uri = oe.utils.squashspaces(d.getVar('SRC_URI') or "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500273 write_recipehistory(rcpinfo, d)
274
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500275 bb.build.exec_func("read_subpackage_metadata", d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500276
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500277 for pkg in packagelist:
278 localdata = d.createCopy()
279 localdata.setVar('OVERRIDES', d.getVar("OVERRIDES", False) + ":" + pkg)
280
281 pkge = localdata.getVar("PKGE") or '0'
282 pkgv = localdata.getVar("PKGV")
283 pkgr = localdata.getVar("PKGR")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500284 #
285 # Find out what the last version was
286 # Make sure the version did not decrease
287 #
288 lastversion = getlastpkgversion(pkg)
289 if lastversion:
290 last_pkge = lastversion.pkge
291 last_pkgv = lastversion.pkgv
292 last_pkgr = lastversion.pkgr
293 r = bb.utils.vercmp((pkge, pkgv, pkgr), (last_pkge, last_pkgv, last_pkgr))
294 if r < 0:
Brad Bishop1d80a2e2019-11-15 16:35:03 -0500295 msg = "Package version for package %s went backwards which would break package feeds (from %s:%s-%s to %s:%s-%s)" % (pkg, last_pkge, last_pkgv, last_pkgr, pkge, pkgv, pkgr)
Andrew Geisslereff27472021-10-29 15:35:00 -0500296 oe.qa.handle_error("version-going-backwards", msg, d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500297
298 pkginfo = PackageInfo(pkg)
299 # Apparently the version can be different on a per-package basis (see Python)
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500300 pkginfo.pe = localdata.getVar("PE") or '0'
301 pkginfo.pv = localdata.getVar("PV")
302 pkginfo.pr = localdata.getVar("PR")
303 pkginfo.pkg = localdata.getVar("PKG")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500304 pkginfo.pkge = pkge
305 pkginfo.pkgv = pkgv
306 pkginfo.pkgr = pkgr
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500307 pkginfo.rprovides = sortpkglist(oe.utils.squashspaces(localdata.getVar("RPROVIDES") or ""))
308 pkginfo.rdepends = sortpkglist(oe.utils.squashspaces(localdata.getVar("RDEPENDS") or ""))
309 pkginfo.rrecommends = sortpkglist(oe.utils.squashspaces(localdata.getVar("RRECOMMENDS") or ""))
310 pkginfo.rsuggests = sortpkglist(oe.utils.squashspaces(localdata.getVar("RSUGGESTS") or ""))
311 pkginfo.replaces = sortpkglist(oe.utils.squashspaces(localdata.getVar("RREPLACES") or ""))
312 pkginfo.rconflicts = sortpkglist(oe.utils.squashspaces(localdata.getVar("RCONFLICTS") or ""))
313 pkginfo.files = oe.utils.squashspaces(localdata.getVar("FILES") or "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500314 for filevar in pkginfo.filevars:
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500315 pkginfo.filevars[filevar] = localdata.getVar(filevar) or ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500316
317 # Gather information about packaged files
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500318 val = localdata.getVar('FILES_INFO') or ''
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500319 dictval = json.loads(val)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600320 filelist = list(dictval.keys())
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500321 filelist.sort()
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800322 pkginfo.filelist = " ".join([shlex.quote(x) for x in filelist])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500323
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500324 pkginfo.size = int(localdata.getVar('PKGSIZE') or '0')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500325
326 write_pkghistory(pkginfo, d)
327
Andrew Geisslereff27472021-10-29 15:35:00 -0500328 oe.qa.exit_if_errors(d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500329}
330
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500331python buildhistory_emit_outputsigs() {
332 if not "task" in (d.getVar('BUILDHISTORY_FEATURES') or "").split():
333 return
334
335 import hashlib
336
337 taskoutdir = os.path.join(d.getVar('BUILDHISTORY_DIR'), 'task', 'output')
338 bb.utils.mkdirhier(taskoutdir)
339 currenttask = d.getVar('BB_CURRENTTASK')
340 pn = d.getVar('PN')
341 taskfile = os.path.join(taskoutdir, '%s.%s' % (pn, currenttask))
342
343 cwd = os.getcwd()
344 filesigs = {}
345 for root, _, files in os.walk(cwd):
346 for fname in files:
347 if fname == 'fixmepath':
348 continue
349 fullpath = os.path.join(root, fname)
350 try:
351 if os.path.islink(fullpath):
352 sha256 = hashlib.sha256(os.readlink(fullpath).encode('utf-8')).hexdigest()
353 elif os.path.isfile(fullpath):
354 sha256 = bb.utils.sha256_file(fullpath)
355 else:
356 continue
357 except OSError:
358 bb.warn('buildhistory: unable to read %s to get output signature' % fullpath)
359 continue
360 filesigs[os.path.relpath(fullpath, cwd)] = sha256
361 with open(taskfile, 'w') as f:
362 for fpath, fsig in sorted(filesigs.items(), key=lambda item: item[0]):
363 f.write('%s %s\n' % (fpath, fsig))
364}
365
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500366
367def write_recipehistory(rcpinfo, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500368 bb.debug(2, "Writing recipe history")
369
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500370 pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500371
372 infofile = os.path.join(pkghistdir, "latest")
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600373 with open(infofile, "w") as f:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500374 if rcpinfo.pe != "0":
375 f.write(u"PE = %s\n" % rcpinfo.pe)
376 f.write(u"PV = %s\n" % rcpinfo.pv)
377 f.write(u"PR = %s\n" % rcpinfo.pr)
378 f.write(u"DEPENDS = %s\n" % rcpinfo.depends)
379 f.write(u"PACKAGES = %s\n" % rcpinfo.packages)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500380 f.write(u"LAYER = %s\n" % rcpinfo.layer)
Andrew Geissler6ce62a22020-11-30 19:58:47 -0600381 f.write(u"LICENSE = %s\n" % rcpinfo.license)
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500382 f.write(u"CONFIG = %s\n" % rcpinfo.config)
Andrew Geissler4c19ea12020-10-27 13:52:24 -0500383 f.write(u"SRC_URI = %s\n" % rcpinfo.src_uri)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500384
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500385 write_latest_srcrev(d, pkghistdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500386
387def write_pkghistory(pkginfo, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500388 bb.debug(2, "Writing package history for package %s" % pkginfo.name)
389
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500390 pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500391
392 pkgpath = os.path.join(pkghistdir, pkginfo.name)
393 if not os.path.exists(pkgpath):
394 bb.utils.mkdirhier(pkgpath)
395
396 infofile = os.path.join(pkgpath, "latest")
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600397 with open(infofile, "w") as f:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500398 if pkginfo.pe != "0":
399 f.write(u"PE = %s\n" % pkginfo.pe)
400 f.write(u"PV = %s\n" % pkginfo.pv)
401 f.write(u"PR = %s\n" % pkginfo.pr)
402
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600403 if pkginfo.pkg != pkginfo.name:
404 f.write(u"PKG = %s\n" % pkginfo.pkg)
405 if pkginfo.pkge != pkginfo.pe:
406 f.write(u"PKGE = %s\n" % pkginfo.pkge)
407 if pkginfo.pkgv != pkginfo.pv:
408 f.write(u"PKGV = %s\n" % pkginfo.pkgv)
409 if pkginfo.pkgr != pkginfo.pr:
410 f.write(u"PKGR = %s\n" % pkginfo.pkgr)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500411 f.write(u"RPROVIDES = %s\n" % pkginfo.rprovides)
412 f.write(u"RDEPENDS = %s\n" % pkginfo.rdepends)
413 f.write(u"RRECOMMENDS = %s\n" % pkginfo.rrecommends)
414 if pkginfo.rsuggests:
415 f.write(u"RSUGGESTS = %s\n" % pkginfo.rsuggests)
416 if pkginfo.rreplaces:
417 f.write(u"RREPLACES = %s\n" % pkginfo.rreplaces)
418 if pkginfo.rconflicts:
419 f.write(u"RCONFLICTS = %s\n" % pkginfo.rconflicts)
420 f.write(u"PKGSIZE = %d\n" % pkginfo.size)
421 f.write(u"FILES = %s\n" % pkginfo.files)
422 f.write(u"FILELIST = %s\n" % pkginfo.filelist)
423
424 for filevar in pkginfo.filevars:
425 filevarpath = os.path.join(pkgpath, "latest.%s" % filevar)
426 val = pkginfo.filevars[filevar]
427 if val:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600428 with open(filevarpath, "w") as f:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500429 f.write(val)
430 else:
431 if os.path.exists(filevarpath):
432 os.unlink(filevarpath)
433
434#
435# rootfs_type can be: image, sdk_target, sdk_host
436#
437def buildhistory_list_installed(d, rootfs_type="image"):
438 from oe.rootfs import image_list_installed_packages
439 from oe.sdk import sdk_list_installed_packages
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500440 from oe.utils import format_pkg_list
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500441
Andrew Geissler635e0e42020-08-21 15:58:33 -0500442 process_list = [('file', 'bh_installed_pkgs_%s.txt' % os.getpid()),\
443 ('deps', 'bh_installed_pkgs_deps_%s.txt' % os.getpid())]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500444
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500445 if rootfs_type == "image":
446 pkgs = image_list_installed_packages(d)
447 else:
448 pkgs = sdk_list_installed_packages(d, rootfs_type == "sdk_target")
449
Andrew Geisslereff27472021-10-29 15:35:00 -0500450 if rootfs_type == "sdk_host":
451 pkgdata_dir = d.getVar('PKGDATA_DIR_SDK')
452 else:
453 pkgdata_dir = d.getVar('PKGDATA_DIR')
454
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500455 for output_type, output_file in process_list:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500456 output_file_full = os.path.join(d.getVar('WORKDIR'), output_file)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500457
458 with open(output_file_full, 'w') as output:
Andrew Geisslereff27472021-10-29 15:35:00 -0500459 output.write(format_pkg_list(pkgs, output_type, pkgdata_dir))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500460
461python buildhistory_list_installed_image() {
462 buildhistory_list_installed(d)
463}
464
465python buildhistory_list_installed_sdk_target() {
466 buildhistory_list_installed(d, "sdk_target")
467}
468
469python buildhistory_list_installed_sdk_host() {
470 buildhistory_list_installed(d, "sdk_host")
471}
472
473buildhistory_get_installed() {
474 mkdir -p $1
475
476 # Get list of installed packages
477 pkgcache="$1/installed-packages.tmp"
Andrew Geissler635e0e42020-08-21 15:58:33 -0500478 cat ${WORKDIR}/bh_installed_pkgs_${PID}.txt | sort > $pkgcache && rm ${WORKDIR}/bh_installed_pkgs_${PID}.txt
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500479
480 cat $pkgcache | awk '{ print $1 }' > $1/installed-package-names.txt
Andrew Geissler635e0e42020-08-21 15:58:33 -0500481
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500482 if [ -s $pkgcache ] ; then
483 cat $pkgcache | awk '{ print $2 }' | xargs -n1 basename > $1/installed-packages.txt
484 else
485 printf "" > $1/installed-packages.txt
486 fi
487
488 # Produce dependency graph
489 # First, quote each name to handle characters that cause issues for dot
Andrew Geissler635e0e42020-08-21 15:58:33 -0500490 sed 's:\([^| ]*\):"\1":g' ${WORKDIR}/bh_installed_pkgs_deps_${PID}.txt > $1/depends.tmp &&
491 rm ${WORKDIR}/bh_installed_pkgs_deps_${PID}.txt
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500492 # Remove lines with rpmlib(...) and config(...) dependencies, change the
493 # delimiter from pipe to "->", set the style for recommend lines and
494 # turn versioned dependencies into edge labels.
495 sed -i -e '/rpmlib(/d' \
496 -e '/config(/d' \
497 -e 's:|: -> :' \
498 -e 's:"\[REC\]":[style=dotted]:' \
499 -e 's:"\([<>=]\+\)" "\([^"]*\)":[label="\1 \2"]:' \
Andrew Geisslerd159c7f2021-09-02 21:05:58 -0500500 -e 's:"\([*]\+\)" "\([^"]*\)":[label="\2"]:' \
501 -e 's:"\[RPROVIDES\]":[style=dashed]:' \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500502 $1/depends.tmp
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500503 # Add header, sorted and de-duped contents and footer and then delete the temp file
504 printf "digraph depends {\n node [shape=plaintext]\n" > $1/depends.dot
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500505 cat $1/depends.tmp | sort -u >> $1/depends.dot
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500506 echo "}" >> $1/depends.dot
507 rm $1/depends.tmp
508
Andrew Geisslereff27472021-10-29 15:35:00 -0500509 # Set correct pkgdatadir
510 pkgdatadir=${PKGDATA_DIR}
511 if [ "$2" == "sdk" ] && [ "$3" == "host" ]; then
512 pkgdatadir="${PKGDATA_DIR_SDK}"
513 fi
514
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500515 # Produce installed package sizes list
Andrew Geisslereff27472021-10-29 15:35:00 -0500516 oe-pkgdata-util -p $pkgdatadir read-value "PKGSIZE" -n -f $pkgcache > $1/installed-package-sizes.tmp
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500517 cat $1/installed-package-sizes.tmp | awk '{print $2 "\tKiB\t" $1}' | sort -n -r > $1/installed-package-sizes.txt
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500518 rm $1/installed-package-sizes.tmp
519
Andrew Geisslerd159c7f2021-09-02 21:05:58 -0500520 # Produce package info: runtime_name, buildtime_name, recipe, version, size
Andrew Geisslereff27472021-10-29 15:35:00 -0500521 oe-pkgdata-util -p $pkgdatadir read-value "PACKAGE,PN,PV,PKGSIZE" -n -f $pkgcache > $1/installed-package-info.tmp
Andrew Geisslerd159c7f2021-09-02 21:05:58 -0500522 cat $1/installed-package-info.tmp | sort -n -r -k 5 > $1/installed-package-info.txt
523 rm $1/installed-package-info.tmp
524
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500525 # We're now done with the cache, delete it
526 rm $pkgcache
527
528 if [ "$2" != "sdk" ] ; then
529 # Produce some cut-down graphs (for readability)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500530 grep -v kernel-image $1/depends.dot | grep -v kernel-3 | grep -v kernel-4 > $1/depends-nokernel.dot
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500531 grep -v libc6 $1/depends-nokernel.dot | grep -v libgcc > $1/depends-nokernel-nolibc.dot
532 grep -v update- $1/depends-nokernel-nolibc.dot > $1/depends-nokernel-nolibc-noupdate.dot
533 grep -v kernel-module $1/depends-nokernel-nolibc-noupdate.dot > $1/depends-nokernel-nolibc-noupdate-nomodules.dot
534 fi
535
536 # add complementary package information
537 if [ -e ${WORKDIR}/complementary_pkgs.txt ]; then
538 cp ${WORKDIR}/complementary_pkgs.txt $1
539 fi
540}
541
542buildhistory_get_image_installed() {
543 # Anything requiring the use of the packaging system should be done in here
544 # in case the packaging files are going to be removed for this image
545
546 if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'image', '1', '0', d)}" = "0" ] ; then
547 return
548 fi
549
550 buildhistory_get_installed ${BUILDHISTORY_DIR_IMAGE}
551}
552
553buildhistory_get_sdk_installed() {
554 # Anything requiring the use of the packaging system should be done in here
555 # in case the packaging files are going to be removed for this SDK
556
557 if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'sdk', '1', '0', d)}" = "0" ] ; then
558 return
559 fi
560
Andrew Geisslereff27472021-10-29 15:35:00 -0500561 buildhistory_get_installed ${BUILDHISTORY_DIR_SDK}/$1 sdk $1
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500562}
563
564buildhistory_get_sdk_installed_host() {
565 buildhistory_get_sdk_installed host
566}
567
568buildhistory_get_sdk_installed_target() {
569 buildhistory_get_sdk_installed target
570}
571
572buildhistory_list_files() {
573 # List the files in the specified directory, but exclude date/time etc.
Brad Bishop19323692019-04-05 15:28:33 -0400574 # This is somewhat messy, but handles where the size is not printed for device files under pseudo
575 ( cd $1
576 find_cmd='find . ! -path . -printf "%M %-10u %-10g %10s %p -> %l\n"'
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500577 if [ "$3" = "fakeroot" ] ; then
Brad Bishop19323692019-04-05 15:28:33 -0400578 eval ${FAKEROOTENV} ${FAKEROOTCMD} $find_cmd
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500579 else
Brad Bishop19323692019-04-05 15:28:33 -0400580 eval $find_cmd
581 fi | sort -k5 | sed 's/ * -> $//' > $2 )
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500582}
583
Brad Bishop96ff1982019-08-19 13:50:42 -0400584buildhistory_list_files_no_owners() {
585 # List the files in the specified directory, but exclude date/time etc.
586 # Also don't output the ownership data, but instead output just - - so
587 # that the same parsing code as for _list_files works.
588 # This is somewhat messy, but handles where the size is not printed for device files under pseudo
589 ( cd $1
590 find_cmd='find . ! -path . -printf "%M - - %10s %p -> %l\n"'
591 if [ "$3" = "fakeroot" ] ; then
592 eval ${FAKEROOTENV} ${FAKEROOTCMD} "$find_cmd"
593 else
594 eval "$find_cmd"
595 fi | sort -k5 | sed 's/ * -> $//' > $2 )
596}
597
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500598buildhistory_list_pkg_files() {
599 # Create individual files-in-package for each recipe's package
600 for pkgdir in $(find ${PKGDEST}/* -maxdepth 0 -type d); do
601 pkgname=$(basename $pkgdir)
602 outfolder="${BUILDHISTORY_DIR_PACKAGE}/$pkgname"
603 outfile="$outfolder/files-in-package.txt"
604 # Make sure the output folder exists so we can create the file
605 if [ ! -d $outfolder ] ; then
606 bbdebug 2 "Folder $outfolder does not exist, file $outfile not created"
607 continue
608 fi
609 buildhistory_list_files $pkgdir $outfile fakeroot
610 done
611}
612
613buildhistory_get_imageinfo() {
614 if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'image', '1', '0', d)}" = "0" ] ; then
615 return
616 fi
617
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500618 mkdir -p ${BUILDHISTORY_DIR_IMAGE}
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500619 buildhistory_list_files ${IMAGE_ROOTFS} ${BUILDHISTORY_DIR_IMAGE}/files-in-image.txt
620
621 # Collect files requested in BUILDHISTORY_IMAGE_FILES
622 rm -rf ${BUILDHISTORY_DIR_IMAGE}/image-files
623 for f in ${BUILDHISTORY_IMAGE_FILES}; do
624 if [ -f ${IMAGE_ROOTFS}/$f ] ; then
625 mkdir -p ${BUILDHISTORY_DIR_IMAGE}/image-files/`dirname $f`
626 cp ${IMAGE_ROOTFS}/$f ${BUILDHISTORY_DIR_IMAGE}/image-files/$f
627 fi
628 done
629
630 # Record some machine-readable meta-information about the image
631 printf "" > ${BUILDHISTORY_DIR_IMAGE}/image-info.txt
632 cat >> ${BUILDHISTORY_DIR_IMAGE}/image-info.txt <<END
633${@buildhistory_get_imagevars(d)}
634END
635 imagesize=`du -ks ${IMAGE_ROOTFS} | awk '{ print $1 }'`
636 echo "IMAGESIZE = $imagesize" >> ${BUILDHISTORY_DIR_IMAGE}/image-info.txt
637
638 # Add some configuration information
639 echo "${MACHINE}: ${IMAGE_BASENAME} configured for ${DISTRO} ${DISTRO_VERSION}" > ${BUILDHISTORY_DIR_IMAGE}/build-id.txt
640
641 cat >> ${BUILDHISTORY_DIR_IMAGE}/build-id.txt <<END
642${@buildhistory_get_build_id(d)}
643END
644}
645
646buildhistory_get_sdkinfo() {
647 if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'sdk', '1', '0', d)}" = "0" ] ; then
648 return
649 fi
650
651 buildhistory_list_files ${SDK_OUTPUT} ${BUILDHISTORY_DIR_SDK}/files-in-sdk.txt
652
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500653 # Collect files requested in BUILDHISTORY_SDK_FILES
654 rm -rf ${BUILDHISTORY_DIR_SDK}/sdk-files
655 for f in ${BUILDHISTORY_SDK_FILES}; do
656 if [ -f ${SDK_OUTPUT}/${SDKPATH}/$f ] ; then
657 mkdir -p ${BUILDHISTORY_DIR_SDK}/sdk-files/`dirname $f`
658 cp ${SDK_OUTPUT}/${SDKPATH}/$f ${BUILDHISTORY_DIR_SDK}/sdk-files/$f
659 fi
660 done
661
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500662 # Record some machine-readable meta-information about the SDK
663 printf "" > ${BUILDHISTORY_DIR_SDK}/sdk-info.txt
664 cat >> ${BUILDHISTORY_DIR_SDK}/sdk-info.txt <<END
665${@buildhistory_get_sdkvars(d)}
666END
667 sdksize=`du -ks ${SDK_OUTPUT} | awk '{ print $1 }'`
668 echo "SDKSIZE = $sdksize" >> ${BUILDHISTORY_DIR_SDK}/sdk-info.txt
669}
670
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500671python buildhistory_get_extra_sdkinfo() {
672 import operator
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500673 from oe.sdk import get_extra_sdkinfo
674
675 sstate_dir = d.expand('${SDK_OUTPUT}/${SDKPATH}/sstate-cache')
676 extra_info = get_extra_sdkinfo(sstate_dir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500677
678 if d.getVar('BB_CURRENTTASK') == 'populate_sdk_ext' and \
679 "sdk" in (d.getVar('BUILDHISTORY_FEATURES') or "").split():
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500680 with open(d.expand('${BUILDHISTORY_DIR_SDK}/sstate-package-sizes.txt'), 'w') as f:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500681 filesizes_sorted = sorted(extra_info['filesizes'].items(), key=operator.itemgetter(1, 0), reverse=True)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500682 for fn, size in filesizes_sorted:
683 f.write('%10d KiB %s\n' % (size, fn))
684 with open(d.expand('${BUILDHISTORY_DIR_SDK}/sstate-task-sizes.txt'), 'w') as f:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500685 tasksizes_sorted = sorted(extra_info['tasksizes'].items(), key=operator.itemgetter(1, 0), reverse=True)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500686 for task, size in tasksizes_sorted:
687 f.write('%10d KiB %s\n' % (size, task))
688}
689
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500690# By using ROOTFS_POSTUNINSTALL_COMMAND we get in after uninstallation of
691# unneeded packages but before the removal of packaging files
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500692ROOTFS_POSTUNINSTALL_COMMAND += "buildhistory_list_installed_image ;"
693ROOTFS_POSTUNINSTALL_COMMAND += "buildhistory_get_image_installed ;"
694ROOTFS_POSTUNINSTALL_COMMAND[vardepvalueexclude] .= "| buildhistory_list_installed_image ;| buildhistory_get_image_installed ;"
695ROOTFS_POSTUNINSTALL_COMMAND[vardepsexclude] += "buildhistory_list_installed_image buildhistory_get_image_installed"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500696
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500697IMAGE_POSTPROCESS_COMMAND += "buildhistory_get_imageinfo ;"
698IMAGE_POSTPROCESS_COMMAND[vardepvalueexclude] .= "| buildhistory_get_imageinfo ;"
699IMAGE_POSTPROCESS_COMMAND[vardepsexclude] += "buildhistory_get_imageinfo"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500700
701# We want these to be the last run so that we get called after complementary package installation
Patrick Williams213cb262021-08-07 19:21:33 -0500702POPULATE_SDK_POST_TARGET_COMMAND:append = " buildhistory_list_installed_sdk_target;"
703POPULATE_SDK_POST_TARGET_COMMAND:append = " buildhistory_get_sdk_installed_target;"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500704POPULATE_SDK_POST_TARGET_COMMAND[vardepvalueexclude] .= "| buildhistory_list_installed_sdk_target;| buildhistory_get_sdk_installed_target;"
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500705POPULATE_SDK_POST_TARGET_COMMAND[vardepsexclude] += "buildhistory_list_installed_sdk_target buildhistory_get_sdk_installed_target"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500706
Patrick Williams213cb262021-08-07 19:21:33 -0500707POPULATE_SDK_POST_HOST_COMMAND:append = " buildhistory_list_installed_sdk_host;"
708POPULATE_SDK_POST_HOST_COMMAND:append = " buildhistory_get_sdk_installed_host;"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500709POPULATE_SDK_POST_HOST_COMMAND[vardepvalueexclude] .= "| buildhistory_list_installed_sdk_host;| buildhistory_get_sdk_installed_host;"
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500710POPULATE_SDK_POST_HOST_COMMAND[vardepsexclude] += "buildhistory_list_installed_sdk_host buildhistory_get_sdk_installed_host"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500711
Patrick Williams213cb262021-08-07 19:21:33 -0500712SDK_POSTPROCESS_COMMAND:append = " buildhistory_get_sdkinfo ; buildhistory_get_extra_sdkinfo; "
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500713SDK_POSTPROCESS_COMMAND[vardepvalueexclude] .= "| buildhistory_get_sdkinfo ; buildhistory_get_extra_sdkinfo; "
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500714SDK_POSTPROCESS_COMMAND[vardepsexclude] += "buildhistory_get_sdkinfo buildhistory_get_extra_sdkinfo"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500715
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500716python buildhistory_write_sigs() {
717 if not "task" in (d.getVar('BUILDHISTORY_FEATURES') or "").split():
718 return
719
720 # Create sigs file
721 if hasattr(bb.parse.siggen, 'dump_siglist'):
722 taskoutdir = os.path.join(d.getVar('BUILDHISTORY_DIR'), 'task')
723 bb.utils.mkdirhier(taskoutdir)
Andrew Geissler09036742021-06-25 14:25:14 -0500724 bb.parse.siggen.dump_siglist(os.path.join(taskoutdir, 'tasksigs.txt'), d.getVar("BUILDHISTORY_PATH_PREFIX_STRIP"))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500725}
726
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500727def buildhistory_get_build_id(d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500728 if d.getVar('BB_WORKERCONTEXT') != '1':
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500729 return ""
730 localdata = bb.data.createCopy(d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500731 statuslines = []
732 for func in oe.data.typed_value('BUILDCFG_FUNCS', localdata):
733 g = globals()
734 if func not in g:
735 bb.warn("Build configuration function '%s' does not exist" % func)
736 else:
737 flines = g[func](localdata)
738 if flines:
739 statuslines.extend(flines)
740
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500741 statusheader = d.getVar('BUILDCFG_HEADER')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500742 return('\n%s\n%s\n' % (statusheader, '\n'.join(statuslines)))
743
Brad Bishop19323692019-04-05 15:28:33 -0400744def buildhistory_get_modified(path):
745 # copied from get_layer_git_status() in image-buildinfo.bbclass
746 import subprocess
747 try:
748 subprocess.check_output("""cd %s; export PSEUDO_UNLOAD=1; set -e;
749 git diff --quiet --no-ext-diff
750 git diff --quiet --no-ext-diff --cached""" % path,
751 shell=True,
752 stderr=subprocess.STDOUT)
753 return ""
754 except subprocess.CalledProcessError as ex:
755 # Silently treat errors as "modified", without checking for the
756 # (expected) return code 1 in a modified git repo. For example, we get
757 # output and a 129 return code when a layer isn't a git repo at all.
758 return " -- modified"
759
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500760def buildhistory_get_metadata_revs(d):
761 # We want an easily machine-readable format here, so get_layers_branch_rev isn't quite what we want
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500762 layers = (d.getVar("BBLAYERS") or "").split()
Brad Bishop19323692019-04-05 15:28:33 -0400763 medadata_revs = ["%-17s = %s:%s%s" % (os.path.basename(i), \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500764 base_get_metadata_git_branch(i, None).strip(), \
Brad Bishop19323692019-04-05 15:28:33 -0400765 base_get_metadata_git_revision(i, None), \
766 buildhistory_get_modified(i)) \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500767 for i in layers]
768 return '\n'.join(medadata_revs)
769
770def outputvars(vars, listvars, d):
771 vars = vars.split()
772 listvars = listvars.split()
773 ret = ""
774 for var in vars:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500775 value = d.getVar(var) or ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500776 if var in listvars:
777 # Squash out spaces
778 value = oe.utils.squashspaces(value)
779 ret += "%s = %s\n" % (var, value)
780 return ret.rstrip('\n')
781
782def buildhistory_get_imagevars(d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500783 if d.getVar('BB_WORKERCONTEXT') != '1':
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500784 return ""
785 imagevars = "DISTRO DISTRO_VERSION USER_CLASSES IMAGE_CLASSES IMAGE_FEATURES IMAGE_LINGUAS IMAGE_INSTALL BAD_RECOMMENDATIONS NO_RECOMMENDATIONS PACKAGE_EXCLUDE ROOTFS_POSTPROCESS_COMMAND IMAGE_POSTPROCESS_COMMAND"
786 listvars = "USER_CLASSES IMAGE_CLASSES IMAGE_FEATURES IMAGE_LINGUAS IMAGE_INSTALL BAD_RECOMMENDATIONS PACKAGE_EXCLUDE"
787 return outputvars(imagevars, listvars, d)
788
789def buildhistory_get_sdkvars(d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500790 if d.getVar('BB_WORKERCONTEXT') != '1':
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500791 return ""
Andrew Geisslereff27472021-10-29 15:35:00 -0500792 sdkvars = "DISTRO DISTRO_VERSION SDK_NAME SDK_VERSION SDKMACHINE SDKIMAGE_FEATURES TOOLCHAIN_HOST_TASK TOOLCHAIN_TARGET_TASK BAD_RECOMMENDATIONS NO_RECOMMENDATIONS PACKAGE_EXCLUDE"
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500793 if d.getVar('BB_CURRENTTASK') == 'populate_sdk_ext':
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500794 # Extensible SDK uses some additional variables
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600795 sdkvars += " SDK_LOCAL_CONF_WHITELIST SDK_LOCAL_CONF_BLACKLIST SDK_INHERIT_BLACKLIST SDK_UPDATE_URL SDK_EXT_TYPE SDK_RECRDEP_TASKS SDK_INCLUDE_PKGDATA SDK_INCLUDE_TOOLCHAIN"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500796 listvars = "SDKIMAGE_FEATURES BAD_RECOMMENDATIONS PACKAGE_EXCLUDE SDK_LOCAL_CONF_WHITELIST SDK_LOCAL_CONF_BLACKLIST SDK_INHERIT_BLACKLIST"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500797 return outputvars(sdkvars, listvars, d)
798
799
800def buildhistory_get_cmdline(d):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500801 argv = d.getVar('BB_CMDLINE', False)
802 if argv:
803 if argv[0].endswith('bin/bitbake'):
804 bincmd = 'bitbake'
805 else:
806 bincmd = argv[0]
807 return '%s %s' % (bincmd, ' '.join(argv[1:]))
808 return ''
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500809
810
811buildhistory_single_commit() {
812 if [ "$3" = "" ] ; then
813 commitopts="${BUILDHISTORY_DIR}/ --allow-empty"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500814 shortlogprefix="No changes: "
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500815 else
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500816 commitopts=""
817 shortlogprefix=""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500818 fi
819 if [ "${BUILDHISTORY_BUILD_FAILURES}" = "0" ] ; then
820 result="succeeded"
821 else
822 result="failed"
823 fi
824 case ${BUILDHISTORY_BUILD_INTERRUPTED} in
825 1)
826 result="$result (interrupted)"
827 ;;
828 2)
829 result="$result (force interrupted)"
830 ;;
831 esac
832 commitmsgfile=`mktemp`
833 cat > $commitmsgfile << END
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500834${shortlogprefix}Build ${BUILDNAME} of ${DISTRO} ${DISTRO_VERSION} for machine ${MACHINE} on $2
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500835
836cmd: $1
837
838result: $result
839
840metadata revisions:
841END
842 cat ${BUILDHISTORY_DIR}/metadata-revs >> $commitmsgfile
843 git commit $commitopts -F $commitmsgfile --author "${BUILDHISTORY_COMMIT_AUTHOR}" > /dev/null
844 rm $commitmsgfile
845}
846
847buildhistory_commit() {
848 if [ ! -d ${BUILDHISTORY_DIR} ] ; then
849 # Code above that creates this dir never executed, so there can't be anything to commit
850 return
851 fi
852
853 # Create a machine-readable list of metadata revisions for each layer
854 cat > ${BUILDHISTORY_DIR}/metadata-revs <<END
855${@buildhistory_get_metadata_revs(d)}
856END
857
858 ( cd ${BUILDHISTORY_DIR}/
859 # Initialise the repo if necessary
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500860 if [ ! -e .git ] ; then
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500861 git init -q
862 else
Andrew Geissler82c905d2020-04-13 13:39:40 -0500863 git tag -f ${BUILDHISTORY_TAG}-minus-3 ${BUILDHISTORY_TAG}-minus-2 > /dev/null 2>&1 || true
864 git tag -f ${BUILDHISTORY_TAG}-minus-2 ${BUILDHISTORY_TAG}-minus-1 > /dev/null 2>&1 || true
865 git tag -f ${BUILDHISTORY_TAG}-minus-1 > /dev/null 2>&1 || true
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500866 fi
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600867
868 check_git_config
869
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500870 # Check if there are new/changed files to commit (other than metadata-revs)
871 repostatus=`git status --porcelain | grep -v " metadata-revs$"`
872 HOSTNAME=`hostname 2>/dev/null || echo unknown`
873 CMDLINE="${@buildhistory_get_cmdline(d)}"
874 if [ "$repostatus" != "" ] ; then
875 git add -A .
876 # porcelain output looks like "?? packages/foo/bar"
877 # Ensure we commit metadata-revs with the first commit
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500878 buildhistory_single_commit "$CMDLINE" "$HOSTNAME" dummy
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500879 git gc --auto --quiet
880 else
881 buildhistory_single_commit "$CMDLINE" "$HOSTNAME"
882 fi
883 if [ "${BUILDHISTORY_PUSH_REPO}" != "" ] ; then
884 git push -q ${BUILDHISTORY_PUSH_REPO}
885 fi) || true
886}
887
888python buildhistory_eventhandler() {
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600889 if (e.data.getVar('BUILDHISTORY_FEATURES') or "").strip():
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500890 reset = e.data.getVar("BUILDHISTORY_RESET")
891 olddir = e.data.getVar("BUILDHISTORY_OLD_DIR")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500892 if isinstance(e, bb.event.BuildStarted):
893 if reset:
894 import shutil
895 # Clean up after potentially interrupted build.
896 if os.path.isdir(olddir):
897 shutil.rmtree(olddir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500898 rootdir = e.data.getVar("BUILDHISTORY_DIR")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500899 entries = [ x for x in os.listdir(rootdir) if not x.startswith('.') ]
900 bb.utils.mkdirhier(olddir)
901 for entry in entries:
Andrew Geisslerc926e172021-05-07 16:11:35 -0500902 bb.utils.rename(os.path.join(rootdir, entry),
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500903 os.path.join(olddir, entry))
904 elif isinstance(e, bb.event.BuildCompleted):
905 if reset:
906 import shutil
907 shutil.rmtree(olddir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500908 if e.data.getVar("BUILDHISTORY_COMMIT") == "1":
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500909 bb.note("Writing buildhistory")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500910 bb.build.exec_func("buildhistory_write_sigs", d)
Brad Bishopf3fd2882019-06-21 08:06:37 -0400911 import time
912 start=time.time()
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500913 localdata = bb.data.createCopy(e.data)
914 localdata.setVar('BUILDHISTORY_BUILD_FAILURES', str(e._failures))
915 interrupted = getattr(e, '_interrupted', 0)
916 localdata.setVar('BUILDHISTORY_BUILD_INTERRUPTED', str(interrupted))
917 bb.build.exec_func("buildhistory_commit", localdata)
Brad Bishopf3fd2882019-06-21 08:06:37 -0400918 stop=time.time()
919 bb.note("Writing buildhistory took: %s seconds" % round(stop-start))
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500920 else:
921 bb.note("No commit since BUILDHISTORY_COMMIT != '1'")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500922}
923
924addhandler buildhistory_eventhandler
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500925buildhistory_eventhandler[eventmask] = "bb.event.BuildCompleted bb.event.BuildStarted"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500926
927
928# FIXME this ought to be moved into the fetcher
929def _get_srcrev_values(d):
930 """
931 Return the version strings for the current recipe
932 """
933
934 scms = []
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500935 fetcher = bb.fetch.Fetch(d.getVar('SRC_URI').split(), d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500936 urldata = fetcher.ud
937 for u in urldata:
938 if urldata[u].method.supports_srcrev():
939 scms.append(u)
940
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500941 dict_srcrevs = {}
942 dict_tag_srcrevs = {}
943 for scm in scms:
944 ud = urldata[scm]
945 for name in ud.names:
Andrew Geissler595f6302022-01-24 19:11:47 +0000946 autoinc, rev = ud.method.sortable_revision(ud, d, name)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500947 dict_srcrevs[name] = rev
948 if 'tag' in ud.parm:
949 tag = ud.parm['tag'];
950 key = name+'_'+tag
951 dict_tag_srcrevs[key] = rev
952 return (dict_srcrevs, dict_tag_srcrevs)
953
954do_fetch[postfuncs] += "write_srcrev"
955do_fetch[vardepsexclude] += "write_srcrev"
956python write_srcrev() {
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500957 write_latest_srcrev(d, d.getVar('BUILDHISTORY_DIR_PACKAGE'))
958}
959
960def write_latest_srcrev(d, pkghistdir):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500961 srcrevfile = os.path.join(pkghistdir, 'latest_srcrev')
962
963 srcrevs, tag_srcrevs = _get_srcrev_values(d)
964 if srcrevs:
965 if not os.path.exists(pkghistdir):
966 bb.utils.mkdirhier(pkghistdir)
967 old_tag_srcrevs = {}
968 if os.path.exists(srcrevfile):
969 with open(srcrevfile) as f:
970 for line in f:
971 if line.startswith('# tag_'):
972 key, value = line.split("=", 1)
973 key = key.replace('# tag_', '').strip()
974 value = value.replace('"', '').strip()
975 old_tag_srcrevs[key] = value
976 with open(srcrevfile, 'w') as f:
Andrew Geissler595f6302022-01-24 19:11:47 +0000977 for name, srcrev in sorted(srcrevs.items()):
978 suffix = "_" + name
979 if name == "default":
980 suffix = ""
981 orig_srcrev = d.getVar('SRCREV%s' % suffix, False)
982 if orig_srcrev:
983 f.write('# SRCREV%s = "%s"\n' % (suffix, orig_srcrev))
984 f.write('SRCREV%s = "%s"\n' % (suffix, srcrev))
985 for name, srcrev in sorted(tag_srcrevs.items()):
986 f.write('# tag_%s = "%s"\n' % (name, srcrev))
987 if name in old_tag_srcrevs and old_tag_srcrevs[name] != srcrev:
988 pkg = d.getVar('PN')
989 bb.warn("Revision for tag %s in package %s was changed since last build (from %s to %s)" % (name, pkg, old_tag_srcrevs[name], srcrev))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500990
991 else:
992 if os.path.exists(srcrevfile):
993 os.remove(srcrevfile)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500994
995do_testimage[postfuncs] += "write_ptest_result"
996do_testimage[vardepsexclude] += "write_ptest_result"
997
998python write_ptest_result() {
999 write_latest_ptest_result(d, d.getVar('BUILDHISTORY_DIR'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001000}
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001001
1002def write_latest_ptest_result(d, histdir):
1003 import glob
1004 import subprocess
1005 test_log_dir = d.getVar('TEST_LOG_DIR')
1006 input_ptest = os.path.join(test_log_dir, 'ptest_log')
1007 output_ptest = os.path.join(histdir, 'ptest')
1008 if os.path.exists(input_ptest):
1009 try:
1010 # Lock it avoid race issue
1011 lock = bb.utils.lockfile(output_ptest + "/ptest.lock")
1012 bb.utils.mkdirhier(output_ptest)
1013 oe.path.copytree(input_ptest, output_ptest)
1014 # Sort test result
1015 for result in glob.glob('%s/pass.fail.*' % output_ptest):
1016 bb.debug(1, 'Processing %s' % result)
1017 cmd = ['sort', result, '-o', result]
1018 bb.debug(1, 'Running %s' % cmd)
1019 ret = subprocess.call(cmd)
1020 if ret != 0:
1021 bb.error('Failed to run %s!' % cmd)
1022 finally:
1023 bb.utils.unlockfile(lock)