blob: 49af61c9c5cf15aa3b603e5db91d9f40dbd1f983 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001#
2# Records history of build output in order to detect regressions
3#
4# Based in part on testlab.bbclass and packagehistory.bbclass
5#
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05006# Copyright (C) 2011-2016 Intel Corporation
Patrick Williamsc124f4f2015-09-15 14:41:29 -05007# Copyright (C) 2007-2011 Koen Kooi <koen@openembedded.org>
8#
9
Andrew Geisslerc9f78652020-09-18 14:11:35 -050010inherit image-artifact-names
11
Patrick Williamsc124f4f2015-09-15 14:41:29 -050012BUILDHISTORY_FEATURES ?= "image package sdk"
13BUILDHISTORY_DIR ?= "${TOPDIR}/buildhistory"
14BUILDHISTORY_DIR_IMAGE = "${BUILDHISTORY_DIR}/images/${MACHINE_ARCH}/${TCLIBC}/${IMAGE_BASENAME}"
15BUILDHISTORY_DIR_PACKAGE = "${BUILDHISTORY_DIR}/packages/${MULTIMACH_TARGET_SYS}/${PN}"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050016
17# Setting this to non-empty will remove the old content of the buildhistory as part of
18# the current bitbake invocation and replace it with information about what was built
19# during the build.
20#
21# This is meant to be used in continuous integration (CI) systems when invoking bitbake
22# for full world builds. The effect in that case is that information about packages
23# that no longer get build also gets removed from the buildhistory, which is not
24# the case otherwise.
25#
26# The advantage over manually cleaning the buildhistory outside of bitbake is that
27# the "version-going-backwards" check still works. When relying on that, be careful
28# about failed world builds: they will lead to incomplete information in the
29# buildhistory because information about packages that could not be built will
30# also get removed. A CI system should handle that by discarding the buildhistory
31# of failed builds.
32#
33# The expected usage is via auto.conf, but passing via the command line also works
34# with: BB_ENV_EXTRAWHITE=BUILDHISTORY_RESET BUILDHISTORY_RESET=1
35BUILDHISTORY_RESET ?= ""
36
37BUILDHISTORY_OLD_DIR = "${BUILDHISTORY_DIR}/${@ "old" if "${BUILDHISTORY_RESET}" else ""}"
38BUILDHISTORY_OLD_DIR_PACKAGE = "${BUILDHISTORY_OLD_DIR}/packages/${MULTIMACH_TARGET_SYS}/${PN}"
39BUILDHISTORY_DIR_SDK = "${BUILDHISTORY_DIR}/sdk/${SDK_NAME}${SDK_EXT}/${IMAGE_BASENAME}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050040BUILDHISTORY_IMAGE_FILES ?= "/etc/passwd /etc/group"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050041BUILDHISTORY_SDK_FILES ?= "conf/local.conf conf/bblayers.conf conf/auto.conf conf/locked-sigs.inc conf/devtool.conf"
Brad Bishop316dfdd2018-06-25 12:45:53 -040042BUILDHISTORY_COMMIT ?= "1"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050043BUILDHISTORY_COMMIT_AUTHOR ?= "buildhistory <buildhistory@${DISTRO}>"
44BUILDHISTORY_PUSH_REPO ?= ""
Andrew Geissler82c905d2020-04-13 13:39:40 -050045BUILDHISTORY_TAG ?= "build"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050046
47SSTATEPOSTINSTFUNCS_append = " buildhistory_emit_pkghistory"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050048# We want to avoid influencing the signatures of sstate tasks - first the function itself:
Patrick Williamsc124f4f2015-09-15 14:41:29 -050049sstate_install[vardepsexclude] += "buildhistory_emit_pkghistory"
50# then the value added to SSTATEPOSTINSTFUNCS:
51SSTATEPOSTINSTFUNCS[vardepvalueexclude] .= "| buildhistory_emit_pkghistory"
52
Brad Bishop6e60e8b2018-02-01 10:27:11 -050053# Similarly for our function that gets the output signatures
54SSTATEPOSTUNPACKFUNCS_append = " buildhistory_emit_outputsigs"
55sstate_installpkgdir[vardepsexclude] += "buildhistory_emit_outputsigs"
56SSTATEPOSTUNPACKFUNCS[vardepvalueexclude] .= "| buildhistory_emit_outputsigs"
57
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050058# All items excepts those listed here will be removed from a recipe's
59# build history directory by buildhistory_emit_pkghistory(). This is
60# necessary because some of these items (package directories, files that
61# we no longer emit) might be obsolete.
Patrick Williamsc124f4f2015-09-15 14:41:29 -050062#
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050063# When extending build history, derive your class from buildhistory.bbclass
64# and extend this list here with the additional files created by the derived
65# class.
Brad Bishop96ff1982019-08-19 13:50:42 -040066BUILDHISTORY_PRESERVE = "latest latest_srcrev sysroot"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050067
Patrick Williamsc0f7c042017-02-23 20:41:17 -060068PATCH_GIT_USER_EMAIL ?= "buildhistory@oe"
69PATCH_GIT_USER_NAME ?= "OpenEmbedded"
70
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050071#
Brad Bishop96ff1982019-08-19 13:50:42 -040072# Write out the contents of the sysroot
73#
74buildhistory_emit_sysroot() {
75 mkdir --parents ${BUILDHISTORY_DIR_PACKAGE}
76 case ${CLASSOVERRIDE} in
77 class-native|class-cross|class-crosssdk)
78 BASE=${SYSROOT_DESTDIR}/${STAGING_DIR_NATIVE}
79 ;;
80 *)
81 BASE=${SYSROOT_DESTDIR}
82 ;;
83 esac
84 buildhistory_list_files_no_owners $BASE ${BUILDHISTORY_DIR_PACKAGE}/sysroot
85}
86
87#
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050088# Write out metadata about this package for comparison when writing future packages
Patrick Williamsc124f4f2015-09-15 14:41:29 -050089#
90python buildhistory_emit_pkghistory() {
Brad Bishop96ff1982019-08-19 13:50:42 -040091 if d.getVar('BB_CURRENTTASK') in ['populate_sysroot', 'populate_sysroot_setscene']:
92 bb.build.exec_func("buildhistory_emit_sysroot", d)
93
Brad Bishop6e60e8b2018-02-01 10:27:11 -050094 if not d.getVar('BB_CURRENTTASK') in ['packagedata', 'packagedata_setscene']:
Patrick Williamsc124f4f2015-09-15 14:41:29 -050095 return 0
96
Brad Bishop6e60e8b2018-02-01 10:27:11 -050097 if not "package" in (d.getVar('BUILDHISTORY_FEATURES') or "").split():
Patrick Williamsc124f4f2015-09-15 14:41:29 -050098 return 0
99
100 import re
101 import json
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800102 import shlex
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500103 import errno
104
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500105 pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE')
106 oldpkghistdir = d.getVar('BUILDHISTORY_OLD_DIR_PACKAGE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500107
108 class RecipeInfo:
109 def __init__(self, name):
110 self.name = name
111 self.pe = "0"
112 self.pv = "0"
113 self.pr = "r0"
114 self.depends = ""
115 self.packages = ""
116 self.srcrev = ""
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500117 self.layer = ""
Andrew Geissler6ce62a22020-11-30 19:58:47 -0600118 self.license = ""
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500119 self.config = ""
Andrew Geissler4c19ea12020-10-27 13:52:24 -0500120 self.src_uri = ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500121
122
123 class PackageInfo:
124 def __init__(self, name):
125 self.name = name
126 self.pe = "0"
127 self.pv = "0"
128 self.pr = "r0"
129 # pkg/pkge/pkgv/pkgr should be empty because we want to be able to default them
130 self.pkg = ""
131 self.pkge = ""
132 self.pkgv = ""
133 self.pkgr = ""
134 self.size = 0
135 self.depends = ""
136 self.rprovides = ""
137 self.rdepends = ""
138 self.rrecommends = ""
139 self.rsuggests = ""
140 self.rreplaces = ""
141 self.rconflicts = ""
142 self.files = ""
143 self.filelist = ""
144 # Variables that need to be written to their own separate file
145 self.filevars = dict.fromkeys(['pkg_preinst', 'pkg_postinst', 'pkg_prerm', 'pkg_postrm'])
146
147 # Should check PACKAGES here to see if anything removed
148
149 def readPackageInfo(pkg, histfile):
150 pkginfo = PackageInfo(pkg)
151 with open(histfile, "r") as f:
152 for line in f:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500153 lns = line.split('=', 1)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500154 name = lns[0].strip()
155 value = lns[1].strip(" \t\r\n").strip('"')
156 if name == "PE":
157 pkginfo.pe = value
158 elif name == "PV":
159 pkginfo.pv = value
160 elif name == "PR":
161 pkginfo.pr = value
162 elif name == "PKG":
163 pkginfo.pkg = value
164 elif name == "PKGE":
165 pkginfo.pkge = value
166 elif name == "PKGV":
167 pkginfo.pkgv = value
168 elif name == "PKGR":
169 pkginfo.pkgr = value
170 elif name == "RPROVIDES":
171 pkginfo.rprovides = value
172 elif name == "RDEPENDS":
173 pkginfo.rdepends = value
174 elif name == "RRECOMMENDS":
175 pkginfo.rrecommends = value
176 elif name == "RSUGGESTS":
177 pkginfo.rsuggests = value
178 elif name == "RREPLACES":
179 pkginfo.rreplaces = value
180 elif name == "RCONFLICTS":
181 pkginfo.rconflicts = value
182 elif name == "PKGSIZE":
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600183 pkginfo.size = int(value)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500184 elif name == "FILES":
185 pkginfo.files = value
186 elif name == "FILELIST":
187 pkginfo.filelist = value
188 # Apply defaults
189 if not pkginfo.pkg:
190 pkginfo.pkg = pkginfo.name
191 if not pkginfo.pkge:
192 pkginfo.pkge = pkginfo.pe
193 if not pkginfo.pkgv:
194 pkginfo.pkgv = pkginfo.pv
195 if not pkginfo.pkgr:
196 pkginfo.pkgr = pkginfo.pr
197 return pkginfo
198
199 def getlastpkgversion(pkg):
200 try:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500201 histfile = os.path.join(oldpkghistdir, pkg, "latest")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500202 return readPackageInfo(pkg, histfile)
203 except EnvironmentError:
204 return None
205
206 def sortpkglist(string):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500207 pkgiter = re.finditer(r'[a-zA-Z0-9.+-]+( \([><=]+[^)]+\))?', string, 0)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500208 pkglist = [p.group(0) for p in pkgiter]
209 pkglist.sort()
210 return ' '.join(pkglist)
211
212 def sortlist(string):
213 items = string.split(' ')
214 items.sort()
215 return ' '.join(items)
216
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500217 pn = d.getVar('PN')
218 pe = d.getVar('PE') or "0"
219 pv = d.getVar('PV')
220 pr = d.getVar('PR')
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500221 layer = bb.utils.get_file_layer(d.getVar('FILE'), d)
Andrew Geissler6ce62a22020-11-30 19:58:47 -0600222 license = d.getVar('LICENSE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500223
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500224 pkgdata_dir = d.getVar('PKGDATA_DIR')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500225 packages = ""
226 try:
227 with open(os.path.join(pkgdata_dir, pn)) as f:
228 for line in f.readlines():
229 if line.startswith('PACKAGES: '):
230 packages = oe.utils.squashspaces(line.split(': ', 1)[1])
231 break
232 except IOError as e:
233 if e.errno == errno.ENOENT:
234 # Probably a -cross recipe, just ignore
235 return 0
236 else:
237 raise
238
239 packagelist = packages.split()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500240 preserve = d.getVar('BUILDHISTORY_PRESERVE').split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500241 if not os.path.exists(pkghistdir):
242 bb.utils.mkdirhier(pkghistdir)
243 else:
244 # Remove files for packages that no longer exist
245 for item in os.listdir(pkghistdir):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500246 if item not in preserve:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500247 if item not in packagelist:
248 itempath = os.path.join(pkghistdir, item)
249 if os.path.isdir(itempath):
250 for subfile in os.listdir(itempath):
251 os.unlink(os.path.join(itempath, subfile))
252 os.rmdir(itempath)
253 else:
254 os.unlink(itempath)
255
256 rcpinfo = RecipeInfo(pn)
257 rcpinfo.pe = pe
258 rcpinfo.pv = pv
259 rcpinfo.pr = pr
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500260 rcpinfo.depends = sortlist(oe.utils.squashspaces(d.getVar('DEPENDS') or ""))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500261 rcpinfo.packages = packages
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500262 rcpinfo.layer = layer
Andrew Geissler6ce62a22020-11-30 19:58:47 -0600263 rcpinfo.license = license
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500264 rcpinfo.config = sortlist(oe.utils.squashspaces(d.getVar('PACKAGECONFIG') or ""))
Andrew Geissler4c19ea12020-10-27 13:52:24 -0500265 rcpinfo.src_uri = oe.utils.squashspaces(d.getVar('SRC_URI') or "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500266 write_recipehistory(rcpinfo, d)
267
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500268 bb.build.exec_func("read_subpackage_metadata", d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500269
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500270 for pkg in packagelist:
271 localdata = d.createCopy()
272 localdata.setVar('OVERRIDES', d.getVar("OVERRIDES", False) + ":" + pkg)
273
274 pkge = localdata.getVar("PKGE") or '0'
275 pkgv = localdata.getVar("PKGV")
276 pkgr = localdata.getVar("PKGR")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500277 #
278 # Find out what the last version was
279 # Make sure the version did not decrease
280 #
281 lastversion = getlastpkgversion(pkg)
282 if lastversion:
283 last_pkge = lastversion.pkge
284 last_pkgv = lastversion.pkgv
285 last_pkgr = lastversion.pkgr
286 r = bb.utils.vercmp((pkge, pkgv, pkgr), (last_pkge, last_pkgv, last_pkgr))
287 if r < 0:
Brad Bishop1d80a2e2019-11-15 16:35:03 -0500288 msg = "Package version for package %s went backwards which would break package feeds (from %s:%s-%s to %s:%s-%s)" % (pkg, last_pkge, last_pkgv, last_pkgr, pkge, pkgv, pkgr)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500289 package_qa_handle_error("version-going-backwards", msg, d)
290
291 pkginfo = PackageInfo(pkg)
292 # Apparently the version can be different on a per-package basis (see Python)
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500293 pkginfo.pe = localdata.getVar("PE") or '0'
294 pkginfo.pv = localdata.getVar("PV")
295 pkginfo.pr = localdata.getVar("PR")
296 pkginfo.pkg = localdata.getVar("PKG")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500297 pkginfo.pkge = pkge
298 pkginfo.pkgv = pkgv
299 pkginfo.pkgr = pkgr
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500300 pkginfo.rprovides = sortpkglist(oe.utils.squashspaces(localdata.getVar("RPROVIDES") or ""))
301 pkginfo.rdepends = sortpkglist(oe.utils.squashspaces(localdata.getVar("RDEPENDS") or ""))
302 pkginfo.rrecommends = sortpkglist(oe.utils.squashspaces(localdata.getVar("RRECOMMENDS") or ""))
303 pkginfo.rsuggests = sortpkglist(oe.utils.squashspaces(localdata.getVar("RSUGGESTS") or ""))
304 pkginfo.replaces = sortpkglist(oe.utils.squashspaces(localdata.getVar("RREPLACES") or ""))
305 pkginfo.rconflicts = sortpkglist(oe.utils.squashspaces(localdata.getVar("RCONFLICTS") or ""))
306 pkginfo.files = oe.utils.squashspaces(localdata.getVar("FILES") or "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500307 for filevar in pkginfo.filevars:
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500308 pkginfo.filevars[filevar] = localdata.getVar(filevar) or ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500309
310 # Gather information about packaged files
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500311 val = localdata.getVar('FILES_INFO') or ''
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500312 dictval = json.loads(val)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600313 filelist = list(dictval.keys())
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500314 filelist.sort()
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800315 pkginfo.filelist = " ".join([shlex.quote(x) for x in filelist])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500316
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500317 pkginfo.size = int(localdata.getVar('PKGSIZE') or '0')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500318
319 write_pkghistory(pkginfo, d)
320
321 # Create files-in-<package-name>.txt files containing a list of files of each recipe's package
322 bb.build.exec_func("buildhistory_list_pkg_files", d)
323}
324
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500325python buildhistory_emit_outputsigs() {
326 if not "task" in (d.getVar('BUILDHISTORY_FEATURES') or "").split():
327 return
328
329 import hashlib
330
331 taskoutdir = os.path.join(d.getVar('BUILDHISTORY_DIR'), 'task', 'output')
332 bb.utils.mkdirhier(taskoutdir)
333 currenttask = d.getVar('BB_CURRENTTASK')
334 pn = d.getVar('PN')
335 taskfile = os.path.join(taskoutdir, '%s.%s' % (pn, currenttask))
336
337 cwd = os.getcwd()
338 filesigs = {}
339 for root, _, files in os.walk(cwd):
340 for fname in files:
341 if fname == 'fixmepath':
342 continue
343 fullpath = os.path.join(root, fname)
344 try:
345 if os.path.islink(fullpath):
346 sha256 = hashlib.sha256(os.readlink(fullpath).encode('utf-8')).hexdigest()
347 elif os.path.isfile(fullpath):
348 sha256 = bb.utils.sha256_file(fullpath)
349 else:
350 continue
351 except OSError:
352 bb.warn('buildhistory: unable to read %s to get output signature' % fullpath)
353 continue
354 filesigs[os.path.relpath(fullpath, cwd)] = sha256
355 with open(taskfile, 'w') as f:
356 for fpath, fsig in sorted(filesigs.items(), key=lambda item: item[0]):
357 f.write('%s %s\n' % (fpath, fsig))
358}
359
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500360
361def write_recipehistory(rcpinfo, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500362 bb.debug(2, "Writing recipe history")
363
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500364 pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500365
366 infofile = os.path.join(pkghistdir, "latest")
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600367 with open(infofile, "w") as f:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500368 if rcpinfo.pe != "0":
369 f.write(u"PE = %s\n" % rcpinfo.pe)
370 f.write(u"PV = %s\n" % rcpinfo.pv)
371 f.write(u"PR = %s\n" % rcpinfo.pr)
372 f.write(u"DEPENDS = %s\n" % rcpinfo.depends)
373 f.write(u"PACKAGES = %s\n" % rcpinfo.packages)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500374 f.write(u"LAYER = %s\n" % rcpinfo.layer)
Andrew Geissler6ce62a22020-11-30 19:58:47 -0600375 f.write(u"LICENSE = %s\n" % rcpinfo.license)
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500376 f.write(u"CONFIG = %s\n" % rcpinfo.config)
Andrew Geissler4c19ea12020-10-27 13:52:24 -0500377 f.write(u"SRC_URI = %s\n" % rcpinfo.src_uri)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500378
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500379 write_latest_srcrev(d, pkghistdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500380
381def write_pkghistory(pkginfo, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500382 bb.debug(2, "Writing package history for package %s" % pkginfo.name)
383
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500384 pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500385
386 pkgpath = os.path.join(pkghistdir, pkginfo.name)
387 if not os.path.exists(pkgpath):
388 bb.utils.mkdirhier(pkgpath)
389
390 infofile = os.path.join(pkgpath, "latest")
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600391 with open(infofile, "w") as f:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500392 if pkginfo.pe != "0":
393 f.write(u"PE = %s\n" % pkginfo.pe)
394 f.write(u"PV = %s\n" % pkginfo.pv)
395 f.write(u"PR = %s\n" % pkginfo.pr)
396
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600397 if pkginfo.pkg != pkginfo.name:
398 f.write(u"PKG = %s\n" % pkginfo.pkg)
399 if pkginfo.pkge != pkginfo.pe:
400 f.write(u"PKGE = %s\n" % pkginfo.pkge)
401 if pkginfo.pkgv != pkginfo.pv:
402 f.write(u"PKGV = %s\n" % pkginfo.pkgv)
403 if pkginfo.pkgr != pkginfo.pr:
404 f.write(u"PKGR = %s\n" % pkginfo.pkgr)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500405 f.write(u"RPROVIDES = %s\n" % pkginfo.rprovides)
406 f.write(u"RDEPENDS = %s\n" % pkginfo.rdepends)
407 f.write(u"RRECOMMENDS = %s\n" % pkginfo.rrecommends)
408 if pkginfo.rsuggests:
409 f.write(u"RSUGGESTS = %s\n" % pkginfo.rsuggests)
410 if pkginfo.rreplaces:
411 f.write(u"RREPLACES = %s\n" % pkginfo.rreplaces)
412 if pkginfo.rconflicts:
413 f.write(u"RCONFLICTS = %s\n" % pkginfo.rconflicts)
414 f.write(u"PKGSIZE = %d\n" % pkginfo.size)
415 f.write(u"FILES = %s\n" % pkginfo.files)
416 f.write(u"FILELIST = %s\n" % pkginfo.filelist)
417
418 for filevar in pkginfo.filevars:
419 filevarpath = os.path.join(pkgpath, "latest.%s" % filevar)
420 val = pkginfo.filevars[filevar]
421 if val:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600422 with open(filevarpath, "w") as f:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500423 f.write(val)
424 else:
425 if os.path.exists(filevarpath):
426 os.unlink(filevarpath)
427
428#
429# rootfs_type can be: image, sdk_target, sdk_host
430#
431def buildhistory_list_installed(d, rootfs_type="image"):
432 from oe.rootfs import image_list_installed_packages
433 from oe.sdk import sdk_list_installed_packages
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500434 from oe.utils import format_pkg_list
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500435
Andrew Geissler635e0e42020-08-21 15:58:33 -0500436 process_list = [('file', 'bh_installed_pkgs_%s.txt' % os.getpid()),\
437 ('deps', 'bh_installed_pkgs_deps_%s.txt' % os.getpid())]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500438
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500439 if rootfs_type == "image":
440 pkgs = image_list_installed_packages(d)
441 else:
442 pkgs = sdk_list_installed_packages(d, rootfs_type == "sdk_target")
443
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500444 for output_type, output_file in process_list:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500445 output_file_full = os.path.join(d.getVar('WORKDIR'), output_file)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500446
447 with open(output_file_full, 'w') as output:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500448 output.write(format_pkg_list(pkgs, output_type))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500449
450python buildhistory_list_installed_image() {
451 buildhistory_list_installed(d)
452}
453
454python buildhistory_list_installed_sdk_target() {
455 buildhistory_list_installed(d, "sdk_target")
456}
457
458python buildhistory_list_installed_sdk_host() {
459 buildhistory_list_installed(d, "sdk_host")
460}
461
462buildhistory_get_installed() {
463 mkdir -p $1
464
465 # Get list of installed packages
466 pkgcache="$1/installed-packages.tmp"
Andrew Geissler635e0e42020-08-21 15:58:33 -0500467 cat ${WORKDIR}/bh_installed_pkgs_${PID}.txt | sort > $pkgcache && rm ${WORKDIR}/bh_installed_pkgs_${PID}.txt
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500468
469 cat $pkgcache | awk '{ print $1 }' > $1/installed-package-names.txt
Andrew Geissler635e0e42020-08-21 15:58:33 -0500470
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500471 if [ -s $pkgcache ] ; then
472 cat $pkgcache | awk '{ print $2 }' | xargs -n1 basename > $1/installed-packages.txt
473 else
474 printf "" > $1/installed-packages.txt
475 fi
476
477 # Produce dependency graph
478 # First, quote each name to handle characters that cause issues for dot
Andrew Geissler635e0e42020-08-21 15:58:33 -0500479 sed 's:\([^| ]*\):"\1":g' ${WORKDIR}/bh_installed_pkgs_deps_${PID}.txt > $1/depends.tmp &&
480 rm ${WORKDIR}/bh_installed_pkgs_deps_${PID}.txt
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500481 # Remove lines with rpmlib(...) and config(...) dependencies, change the
482 # delimiter from pipe to "->", set the style for recommend lines and
483 # turn versioned dependencies into edge labels.
484 sed -i -e '/rpmlib(/d' \
485 -e '/config(/d' \
486 -e 's:|: -> :' \
487 -e 's:"\[REC\]":[style=dotted]:' \
488 -e 's:"\([<>=]\+\)" "\([^"]*\)":[label="\1 \2"]:' \
489 $1/depends.tmp
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500490 # Add header, sorted and de-duped contents and footer and then delete the temp file
491 printf "digraph depends {\n node [shape=plaintext]\n" > $1/depends.dot
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500492 cat $1/depends.tmp | sort -u >> $1/depends.dot
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500493 echo "}" >> $1/depends.dot
494 rm $1/depends.tmp
495
496 # Produce installed package sizes list
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500497 oe-pkgdata-util -p ${PKGDATA_DIR} read-value "PKGSIZE" -n -f $pkgcache > $1/installed-package-sizes.tmp
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500498 cat $1/installed-package-sizes.tmp | awk '{print $2 "\tKiB\t" $1}' | sort -n -r > $1/installed-package-sizes.txt
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500499 rm $1/installed-package-sizes.tmp
500
501 # We're now done with the cache, delete it
502 rm $pkgcache
503
504 if [ "$2" != "sdk" ] ; then
505 # Produce some cut-down graphs (for readability)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500506 grep -v kernel-image $1/depends.dot | grep -v kernel-3 | grep -v kernel-4 > $1/depends-nokernel.dot
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500507 grep -v libc6 $1/depends-nokernel.dot | grep -v libgcc > $1/depends-nokernel-nolibc.dot
508 grep -v update- $1/depends-nokernel-nolibc.dot > $1/depends-nokernel-nolibc-noupdate.dot
509 grep -v kernel-module $1/depends-nokernel-nolibc-noupdate.dot > $1/depends-nokernel-nolibc-noupdate-nomodules.dot
510 fi
511
512 # add complementary package information
513 if [ -e ${WORKDIR}/complementary_pkgs.txt ]; then
514 cp ${WORKDIR}/complementary_pkgs.txt $1
515 fi
516}
517
518buildhistory_get_image_installed() {
519 # Anything requiring the use of the packaging system should be done in here
520 # in case the packaging files are going to be removed for this image
521
522 if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'image', '1', '0', d)}" = "0" ] ; then
523 return
524 fi
525
526 buildhistory_get_installed ${BUILDHISTORY_DIR_IMAGE}
527}
528
529buildhistory_get_sdk_installed() {
530 # Anything requiring the use of the packaging system should be done in here
531 # in case the packaging files are going to be removed for this SDK
532
533 if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'sdk', '1', '0', d)}" = "0" ] ; then
534 return
535 fi
536
537 buildhistory_get_installed ${BUILDHISTORY_DIR_SDK}/$1 sdk
538}
539
540buildhistory_get_sdk_installed_host() {
541 buildhistory_get_sdk_installed host
542}
543
544buildhistory_get_sdk_installed_target() {
545 buildhistory_get_sdk_installed target
546}
547
548buildhistory_list_files() {
549 # List the files in the specified directory, but exclude date/time etc.
Brad Bishop19323692019-04-05 15:28:33 -0400550 # This is somewhat messy, but handles where the size is not printed for device files under pseudo
551 ( cd $1
552 find_cmd='find . ! -path . -printf "%M %-10u %-10g %10s %p -> %l\n"'
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500553 if [ "$3" = "fakeroot" ] ; then
Brad Bishop19323692019-04-05 15:28:33 -0400554 eval ${FAKEROOTENV} ${FAKEROOTCMD} $find_cmd
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500555 else
Brad Bishop19323692019-04-05 15:28:33 -0400556 eval $find_cmd
557 fi | sort -k5 | sed 's/ * -> $//' > $2 )
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500558}
559
Brad Bishop96ff1982019-08-19 13:50:42 -0400560buildhistory_list_files_no_owners() {
561 # List the files in the specified directory, but exclude date/time etc.
562 # Also don't output the ownership data, but instead output just - - so
563 # that the same parsing code as for _list_files works.
564 # This is somewhat messy, but handles where the size is not printed for device files under pseudo
565 ( cd $1
566 find_cmd='find . ! -path . -printf "%M - - %10s %p -> %l\n"'
567 if [ "$3" = "fakeroot" ] ; then
568 eval ${FAKEROOTENV} ${FAKEROOTCMD} "$find_cmd"
569 else
570 eval "$find_cmd"
571 fi | sort -k5 | sed 's/ * -> $//' > $2 )
572}
573
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500574buildhistory_list_pkg_files() {
575 # Create individual files-in-package for each recipe's package
576 for pkgdir in $(find ${PKGDEST}/* -maxdepth 0 -type d); do
577 pkgname=$(basename $pkgdir)
578 outfolder="${BUILDHISTORY_DIR_PACKAGE}/$pkgname"
579 outfile="$outfolder/files-in-package.txt"
580 # Make sure the output folder exists so we can create the file
581 if [ ! -d $outfolder ] ; then
582 bbdebug 2 "Folder $outfolder does not exist, file $outfile not created"
583 continue
584 fi
585 buildhistory_list_files $pkgdir $outfile fakeroot
586 done
587}
588
589buildhistory_get_imageinfo() {
590 if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'image', '1', '0', d)}" = "0" ] ; then
591 return
592 fi
593
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500594 mkdir -p ${BUILDHISTORY_DIR_IMAGE}
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500595 buildhistory_list_files ${IMAGE_ROOTFS} ${BUILDHISTORY_DIR_IMAGE}/files-in-image.txt
596
597 # Collect files requested in BUILDHISTORY_IMAGE_FILES
598 rm -rf ${BUILDHISTORY_DIR_IMAGE}/image-files
599 for f in ${BUILDHISTORY_IMAGE_FILES}; do
600 if [ -f ${IMAGE_ROOTFS}/$f ] ; then
601 mkdir -p ${BUILDHISTORY_DIR_IMAGE}/image-files/`dirname $f`
602 cp ${IMAGE_ROOTFS}/$f ${BUILDHISTORY_DIR_IMAGE}/image-files/$f
603 fi
604 done
605
606 # Record some machine-readable meta-information about the image
607 printf "" > ${BUILDHISTORY_DIR_IMAGE}/image-info.txt
608 cat >> ${BUILDHISTORY_DIR_IMAGE}/image-info.txt <<END
609${@buildhistory_get_imagevars(d)}
610END
611 imagesize=`du -ks ${IMAGE_ROOTFS} | awk '{ print $1 }'`
612 echo "IMAGESIZE = $imagesize" >> ${BUILDHISTORY_DIR_IMAGE}/image-info.txt
613
614 # Add some configuration information
615 echo "${MACHINE}: ${IMAGE_BASENAME} configured for ${DISTRO} ${DISTRO_VERSION}" > ${BUILDHISTORY_DIR_IMAGE}/build-id.txt
616
617 cat >> ${BUILDHISTORY_DIR_IMAGE}/build-id.txt <<END
618${@buildhistory_get_build_id(d)}
619END
620}
621
622buildhistory_get_sdkinfo() {
623 if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'sdk', '1', '0', d)}" = "0" ] ; then
624 return
625 fi
626
627 buildhistory_list_files ${SDK_OUTPUT} ${BUILDHISTORY_DIR_SDK}/files-in-sdk.txt
628
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500629 # Collect files requested in BUILDHISTORY_SDK_FILES
630 rm -rf ${BUILDHISTORY_DIR_SDK}/sdk-files
631 for f in ${BUILDHISTORY_SDK_FILES}; do
632 if [ -f ${SDK_OUTPUT}/${SDKPATH}/$f ] ; then
633 mkdir -p ${BUILDHISTORY_DIR_SDK}/sdk-files/`dirname $f`
634 cp ${SDK_OUTPUT}/${SDKPATH}/$f ${BUILDHISTORY_DIR_SDK}/sdk-files/$f
635 fi
636 done
637
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500638 # Record some machine-readable meta-information about the SDK
639 printf "" > ${BUILDHISTORY_DIR_SDK}/sdk-info.txt
640 cat >> ${BUILDHISTORY_DIR_SDK}/sdk-info.txt <<END
641${@buildhistory_get_sdkvars(d)}
642END
643 sdksize=`du -ks ${SDK_OUTPUT} | awk '{ print $1 }'`
644 echo "SDKSIZE = $sdksize" >> ${BUILDHISTORY_DIR_SDK}/sdk-info.txt
645}
646
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500647python buildhistory_get_extra_sdkinfo() {
648 import operator
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500649 from oe.sdk import get_extra_sdkinfo
650
651 sstate_dir = d.expand('${SDK_OUTPUT}/${SDKPATH}/sstate-cache')
652 extra_info = get_extra_sdkinfo(sstate_dir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500653
654 if d.getVar('BB_CURRENTTASK') == 'populate_sdk_ext' and \
655 "sdk" in (d.getVar('BUILDHISTORY_FEATURES') or "").split():
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500656 with open(d.expand('${BUILDHISTORY_DIR_SDK}/sstate-package-sizes.txt'), 'w') as f:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500657 filesizes_sorted = sorted(extra_info['filesizes'].items(), key=operator.itemgetter(1, 0), reverse=True)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500658 for fn, size in filesizes_sorted:
659 f.write('%10d KiB %s\n' % (size, fn))
660 with open(d.expand('${BUILDHISTORY_DIR_SDK}/sstate-task-sizes.txt'), 'w') as f:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500661 tasksizes_sorted = sorted(extra_info['tasksizes'].items(), key=operator.itemgetter(1, 0), reverse=True)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500662 for task, size in tasksizes_sorted:
663 f.write('%10d KiB %s\n' % (size, task))
664}
665
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500666# By using ROOTFS_POSTUNINSTALL_COMMAND we get in after uninstallation of
667# unneeded packages but before the removal of packaging files
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500668ROOTFS_POSTUNINSTALL_COMMAND += "buildhistory_list_installed_image ;"
669ROOTFS_POSTUNINSTALL_COMMAND += "buildhistory_get_image_installed ;"
670ROOTFS_POSTUNINSTALL_COMMAND[vardepvalueexclude] .= "| buildhistory_list_installed_image ;| buildhistory_get_image_installed ;"
671ROOTFS_POSTUNINSTALL_COMMAND[vardepsexclude] += "buildhistory_list_installed_image buildhistory_get_image_installed"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500672
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500673IMAGE_POSTPROCESS_COMMAND += "buildhistory_get_imageinfo ;"
674IMAGE_POSTPROCESS_COMMAND[vardepvalueexclude] .= "| buildhistory_get_imageinfo ;"
675IMAGE_POSTPROCESS_COMMAND[vardepsexclude] += "buildhistory_get_imageinfo"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500676
677# We want these to be the last run so that we get called after complementary package installation
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500678POPULATE_SDK_POST_TARGET_COMMAND_append = " buildhistory_list_installed_sdk_target;"
679POPULATE_SDK_POST_TARGET_COMMAND_append = " buildhistory_get_sdk_installed_target;"
680POPULATE_SDK_POST_TARGET_COMMAND[vardepvalueexclude] .= "| buildhistory_list_installed_sdk_target;| buildhistory_get_sdk_installed_target;"
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500681POPULATE_SDK_POST_TARGET_COMMAND[vardepsexclude] += "buildhistory_list_installed_sdk_target buildhistory_get_sdk_installed_target"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500682
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500683POPULATE_SDK_POST_HOST_COMMAND_append = " buildhistory_list_installed_sdk_host;"
684POPULATE_SDK_POST_HOST_COMMAND_append = " buildhistory_get_sdk_installed_host;"
685POPULATE_SDK_POST_HOST_COMMAND[vardepvalueexclude] .= "| buildhistory_list_installed_sdk_host;| buildhistory_get_sdk_installed_host;"
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500686POPULATE_SDK_POST_HOST_COMMAND[vardepsexclude] += "buildhistory_list_installed_sdk_host buildhistory_get_sdk_installed_host"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500687
688SDK_POSTPROCESS_COMMAND_append = " buildhistory_get_sdkinfo ; buildhistory_get_extra_sdkinfo; "
689SDK_POSTPROCESS_COMMAND[vardepvalueexclude] .= "| buildhistory_get_sdkinfo ; buildhistory_get_extra_sdkinfo; "
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500690SDK_POSTPROCESS_COMMAND[vardepsexclude] += "buildhistory_get_sdkinfo buildhistory_get_extra_sdkinfo"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500691
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500692python buildhistory_write_sigs() {
693 if not "task" in (d.getVar('BUILDHISTORY_FEATURES') or "").split():
694 return
695
696 # Create sigs file
697 if hasattr(bb.parse.siggen, 'dump_siglist'):
698 taskoutdir = os.path.join(d.getVar('BUILDHISTORY_DIR'), 'task')
699 bb.utils.mkdirhier(taskoutdir)
700 bb.parse.siggen.dump_siglist(os.path.join(taskoutdir, 'tasksigs.txt'))
701}
702
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500703def buildhistory_get_build_id(d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500704 if d.getVar('BB_WORKERCONTEXT') != '1':
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500705 return ""
706 localdata = bb.data.createCopy(d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500707 statuslines = []
708 for func in oe.data.typed_value('BUILDCFG_FUNCS', localdata):
709 g = globals()
710 if func not in g:
711 bb.warn("Build configuration function '%s' does not exist" % func)
712 else:
713 flines = g[func](localdata)
714 if flines:
715 statuslines.extend(flines)
716
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500717 statusheader = d.getVar('BUILDCFG_HEADER')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500718 return('\n%s\n%s\n' % (statusheader, '\n'.join(statuslines)))
719
Brad Bishop19323692019-04-05 15:28:33 -0400720def buildhistory_get_modified(path):
721 # copied from get_layer_git_status() in image-buildinfo.bbclass
722 import subprocess
723 try:
724 subprocess.check_output("""cd %s; export PSEUDO_UNLOAD=1; set -e;
725 git diff --quiet --no-ext-diff
726 git diff --quiet --no-ext-diff --cached""" % path,
727 shell=True,
728 stderr=subprocess.STDOUT)
729 return ""
730 except subprocess.CalledProcessError as ex:
731 # Silently treat errors as "modified", without checking for the
732 # (expected) return code 1 in a modified git repo. For example, we get
733 # output and a 129 return code when a layer isn't a git repo at all.
734 return " -- modified"
735
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500736def buildhistory_get_metadata_revs(d):
737 # We want an easily machine-readable format here, so get_layers_branch_rev isn't quite what we want
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500738 layers = (d.getVar("BBLAYERS") or "").split()
Brad Bishop19323692019-04-05 15:28:33 -0400739 medadata_revs = ["%-17s = %s:%s%s" % (os.path.basename(i), \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500740 base_get_metadata_git_branch(i, None).strip(), \
Brad Bishop19323692019-04-05 15:28:33 -0400741 base_get_metadata_git_revision(i, None), \
742 buildhistory_get_modified(i)) \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500743 for i in layers]
744 return '\n'.join(medadata_revs)
745
746def outputvars(vars, listvars, d):
747 vars = vars.split()
748 listvars = listvars.split()
749 ret = ""
750 for var in vars:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500751 value = d.getVar(var) or ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500752 if var in listvars:
753 # Squash out spaces
754 value = oe.utils.squashspaces(value)
755 ret += "%s = %s\n" % (var, value)
756 return ret.rstrip('\n')
757
758def buildhistory_get_imagevars(d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500759 if d.getVar('BB_WORKERCONTEXT') != '1':
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500760 return ""
761 imagevars = "DISTRO DISTRO_VERSION USER_CLASSES IMAGE_CLASSES IMAGE_FEATURES IMAGE_LINGUAS IMAGE_INSTALL BAD_RECOMMENDATIONS NO_RECOMMENDATIONS PACKAGE_EXCLUDE ROOTFS_POSTPROCESS_COMMAND IMAGE_POSTPROCESS_COMMAND"
762 listvars = "USER_CLASSES IMAGE_CLASSES IMAGE_FEATURES IMAGE_LINGUAS IMAGE_INSTALL BAD_RECOMMENDATIONS PACKAGE_EXCLUDE"
763 return outputvars(imagevars, listvars, d)
764
765def buildhistory_get_sdkvars(d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500766 if d.getVar('BB_WORKERCONTEXT') != '1':
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500767 return ""
768 sdkvars = "DISTRO DISTRO_VERSION SDK_NAME SDK_VERSION SDKMACHINE SDKIMAGE_FEATURES BAD_RECOMMENDATIONS NO_RECOMMENDATIONS PACKAGE_EXCLUDE"
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500769 if d.getVar('BB_CURRENTTASK') == 'populate_sdk_ext':
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500770 # Extensible SDK uses some additional variables
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600771 sdkvars += " SDK_LOCAL_CONF_WHITELIST SDK_LOCAL_CONF_BLACKLIST SDK_INHERIT_BLACKLIST SDK_UPDATE_URL SDK_EXT_TYPE SDK_RECRDEP_TASKS SDK_INCLUDE_PKGDATA SDK_INCLUDE_TOOLCHAIN"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500772 listvars = "SDKIMAGE_FEATURES BAD_RECOMMENDATIONS PACKAGE_EXCLUDE SDK_LOCAL_CONF_WHITELIST SDK_LOCAL_CONF_BLACKLIST SDK_INHERIT_BLACKLIST"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500773 return outputvars(sdkvars, listvars, d)
774
775
776def buildhistory_get_cmdline(d):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500777 argv = d.getVar('BB_CMDLINE', False)
778 if argv:
779 if argv[0].endswith('bin/bitbake'):
780 bincmd = 'bitbake'
781 else:
782 bincmd = argv[0]
783 return '%s %s' % (bincmd, ' '.join(argv[1:]))
784 return ''
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500785
786
787buildhistory_single_commit() {
788 if [ "$3" = "" ] ; then
789 commitopts="${BUILDHISTORY_DIR}/ --allow-empty"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500790 shortlogprefix="No changes: "
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500791 else
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500792 commitopts=""
793 shortlogprefix=""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500794 fi
795 if [ "${BUILDHISTORY_BUILD_FAILURES}" = "0" ] ; then
796 result="succeeded"
797 else
798 result="failed"
799 fi
800 case ${BUILDHISTORY_BUILD_INTERRUPTED} in
801 1)
802 result="$result (interrupted)"
803 ;;
804 2)
805 result="$result (force interrupted)"
806 ;;
807 esac
808 commitmsgfile=`mktemp`
809 cat > $commitmsgfile << END
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500810${shortlogprefix}Build ${BUILDNAME} of ${DISTRO} ${DISTRO_VERSION} for machine ${MACHINE} on $2
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500811
812cmd: $1
813
814result: $result
815
816metadata revisions:
817END
818 cat ${BUILDHISTORY_DIR}/metadata-revs >> $commitmsgfile
819 git commit $commitopts -F $commitmsgfile --author "${BUILDHISTORY_COMMIT_AUTHOR}" > /dev/null
820 rm $commitmsgfile
821}
822
823buildhistory_commit() {
824 if [ ! -d ${BUILDHISTORY_DIR} ] ; then
825 # Code above that creates this dir never executed, so there can't be anything to commit
826 return
827 fi
828
829 # Create a machine-readable list of metadata revisions for each layer
830 cat > ${BUILDHISTORY_DIR}/metadata-revs <<END
831${@buildhistory_get_metadata_revs(d)}
832END
833
834 ( cd ${BUILDHISTORY_DIR}/
835 # Initialise the repo if necessary
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500836 if [ ! -e .git ] ; then
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500837 git init -q
838 else
Andrew Geissler82c905d2020-04-13 13:39:40 -0500839 git tag -f ${BUILDHISTORY_TAG}-minus-3 ${BUILDHISTORY_TAG}-minus-2 > /dev/null 2>&1 || true
840 git tag -f ${BUILDHISTORY_TAG}-minus-2 ${BUILDHISTORY_TAG}-minus-1 > /dev/null 2>&1 || true
841 git tag -f ${BUILDHISTORY_TAG}-minus-1 > /dev/null 2>&1 || true
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500842 fi
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600843
844 check_git_config
845
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500846 # Check if there are new/changed files to commit (other than metadata-revs)
847 repostatus=`git status --porcelain | grep -v " metadata-revs$"`
848 HOSTNAME=`hostname 2>/dev/null || echo unknown`
849 CMDLINE="${@buildhistory_get_cmdline(d)}"
850 if [ "$repostatus" != "" ] ; then
851 git add -A .
852 # porcelain output looks like "?? packages/foo/bar"
853 # Ensure we commit metadata-revs with the first commit
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500854 buildhistory_single_commit "$CMDLINE" "$HOSTNAME" dummy
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500855 git gc --auto --quiet
856 else
857 buildhistory_single_commit "$CMDLINE" "$HOSTNAME"
858 fi
859 if [ "${BUILDHISTORY_PUSH_REPO}" != "" ] ; then
860 git push -q ${BUILDHISTORY_PUSH_REPO}
861 fi) || true
862}
863
864python buildhistory_eventhandler() {
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600865 if (e.data.getVar('BUILDHISTORY_FEATURES') or "").strip():
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500866 reset = e.data.getVar("BUILDHISTORY_RESET")
867 olddir = e.data.getVar("BUILDHISTORY_OLD_DIR")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500868 if isinstance(e, bb.event.BuildStarted):
869 if reset:
870 import shutil
871 # Clean up after potentially interrupted build.
872 if os.path.isdir(olddir):
873 shutil.rmtree(olddir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500874 rootdir = e.data.getVar("BUILDHISTORY_DIR")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500875 entries = [ x for x in os.listdir(rootdir) if not x.startswith('.') ]
876 bb.utils.mkdirhier(olddir)
877 for entry in entries:
878 os.rename(os.path.join(rootdir, entry),
879 os.path.join(olddir, entry))
880 elif isinstance(e, bb.event.BuildCompleted):
881 if reset:
882 import shutil
883 shutil.rmtree(olddir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500884 if e.data.getVar("BUILDHISTORY_COMMIT") == "1":
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500885 bb.note("Writing buildhistory")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500886 bb.build.exec_func("buildhistory_write_sigs", d)
Brad Bishopf3fd2882019-06-21 08:06:37 -0400887 import time
888 start=time.time()
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500889 localdata = bb.data.createCopy(e.data)
890 localdata.setVar('BUILDHISTORY_BUILD_FAILURES', str(e._failures))
891 interrupted = getattr(e, '_interrupted', 0)
892 localdata.setVar('BUILDHISTORY_BUILD_INTERRUPTED', str(interrupted))
893 bb.build.exec_func("buildhistory_commit", localdata)
Brad Bishopf3fd2882019-06-21 08:06:37 -0400894 stop=time.time()
895 bb.note("Writing buildhistory took: %s seconds" % round(stop-start))
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500896 else:
897 bb.note("No commit since BUILDHISTORY_COMMIT != '1'")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500898}
899
900addhandler buildhistory_eventhandler
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500901buildhistory_eventhandler[eventmask] = "bb.event.BuildCompleted bb.event.BuildStarted"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500902
903
904# FIXME this ought to be moved into the fetcher
905def _get_srcrev_values(d):
906 """
907 Return the version strings for the current recipe
908 """
909
910 scms = []
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500911 fetcher = bb.fetch.Fetch(d.getVar('SRC_URI').split(), d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500912 urldata = fetcher.ud
913 for u in urldata:
914 if urldata[u].method.supports_srcrev():
915 scms.append(u)
916
917 autoinc_templ = 'AUTOINC+'
918 dict_srcrevs = {}
919 dict_tag_srcrevs = {}
920 for scm in scms:
921 ud = urldata[scm]
922 for name in ud.names:
923 try:
924 rev = ud.method.sortable_revision(ud, d, name)
925 except TypeError:
926 # support old bitbake versions
927 rev = ud.method.sortable_revision(scm, ud, d, name)
928 # Clean this up when we next bump bitbake version
929 if type(rev) != str:
930 autoinc, rev = rev
931 elif rev.startswith(autoinc_templ):
932 rev = rev[len(autoinc_templ):]
933 dict_srcrevs[name] = rev
934 if 'tag' in ud.parm:
935 tag = ud.parm['tag'];
936 key = name+'_'+tag
937 dict_tag_srcrevs[key] = rev
938 return (dict_srcrevs, dict_tag_srcrevs)
939
940do_fetch[postfuncs] += "write_srcrev"
941do_fetch[vardepsexclude] += "write_srcrev"
942python write_srcrev() {
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500943 write_latest_srcrev(d, d.getVar('BUILDHISTORY_DIR_PACKAGE'))
944}
945
946def write_latest_srcrev(d, pkghistdir):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500947 srcrevfile = os.path.join(pkghistdir, 'latest_srcrev')
948
949 srcrevs, tag_srcrevs = _get_srcrev_values(d)
950 if srcrevs:
951 if not os.path.exists(pkghistdir):
952 bb.utils.mkdirhier(pkghistdir)
953 old_tag_srcrevs = {}
954 if os.path.exists(srcrevfile):
955 with open(srcrevfile) as f:
956 for line in f:
957 if line.startswith('# tag_'):
958 key, value = line.split("=", 1)
959 key = key.replace('# tag_', '').strip()
960 value = value.replace('"', '').strip()
961 old_tag_srcrevs[key] = value
962 with open(srcrevfile, 'w') as f:
963 orig_srcrev = d.getVar('SRCREV', False) or 'INVALID'
964 if orig_srcrev != 'INVALID':
965 f.write('# SRCREV = "%s"\n' % orig_srcrev)
966 if len(srcrevs) > 1:
Brad Bishop19323692019-04-05 15:28:33 -0400967 for name, srcrev in sorted(srcrevs.items()):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500968 orig_srcrev = d.getVar('SRCREV_%s' % name, False)
969 if orig_srcrev:
970 f.write('# SRCREV_%s = "%s"\n' % (name, orig_srcrev))
971 f.write('SRCREV_%s = "%s"\n' % (name, srcrev))
972 else:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500973 f.write('SRCREV = "%s"\n' % next(iter(srcrevs.values())))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500974 if len(tag_srcrevs) > 0:
Brad Bishop19323692019-04-05 15:28:33 -0400975 for name, srcrev in sorted(tag_srcrevs.items()):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500976 f.write('# tag_%s = "%s"\n' % (name, srcrev))
977 if name in old_tag_srcrevs and old_tag_srcrevs[name] != srcrev:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500978 pkg = d.getVar('PN')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500979 bb.warn("Revision for tag %s in package %s was changed since last build (from %s to %s)" % (name, pkg, old_tag_srcrevs[name], srcrev))
980
981 else:
982 if os.path.exists(srcrevfile):
983 os.remove(srcrevfile)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500984
985do_testimage[postfuncs] += "write_ptest_result"
986do_testimage[vardepsexclude] += "write_ptest_result"
987
988python write_ptest_result() {
989 write_latest_ptest_result(d, d.getVar('BUILDHISTORY_DIR'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500990}
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500991
992def write_latest_ptest_result(d, histdir):
993 import glob
994 import subprocess
995 test_log_dir = d.getVar('TEST_LOG_DIR')
996 input_ptest = os.path.join(test_log_dir, 'ptest_log')
997 output_ptest = os.path.join(histdir, 'ptest')
998 if os.path.exists(input_ptest):
999 try:
1000 # Lock it avoid race issue
1001 lock = bb.utils.lockfile(output_ptest + "/ptest.lock")
1002 bb.utils.mkdirhier(output_ptest)
1003 oe.path.copytree(input_ptest, output_ptest)
1004 # Sort test result
1005 for result in glob.glob('%s/pass.fail.*' % output_ptest):
1006 bb.debug(1, 'Processing %s' % result)
1007 cmd = ['sort', result, '-o', result]
1008 bb.debug(1, 'Running %s' % cmd)
1009 ret = subprocess.call(cmd)
1010 if ret != 0:
1011 bb.error('Failed to run %s!' % cmd)
1012 finally:
1013 bb.utils.unlockfile(lock)