blob: 5099e70fb7b51ddbfad00b304c025129ea47f96d [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001#
2# Records history of build output in order to detect regressions
3#
4# Based in part on testlab.bbclass and packagehistory.bbclass
5#
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05006# Copyright (C) 2011-2016 Intel Corporation
Patrick Williamsc124f4f2015-09-15 14:41:29 -05007# Copyright (C) 2007-2011 Koen Kooi <koen@openembedded.org>
8#
9
Andrew Geisslerc9f78652020-09-18 14:11:35 -050010inherit image-artifact-names
11
Patrick Williamsc124f4f2015-09-15 14:41:29 -050012BUILDHISTORY_FEATURES ?= "image package sdk"
13BUILDHISTORY_DIR ?= "${TOPDIR}/buildhistory"
14BUILDHISTORY_DIR_IMAGE = "${BUILDHISTORY_DIR}/images/${MACHINE_ARCH}/${TCLIBC}/${IMAGE_BASENAME}"
15BUILDHISTORY_DIR_PACKAGE = "${BUILDHISTORY_DIR}/packages/${MULTIMACH_TARGET_SYS}/${PN}"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050016
17# Setting this to non-empty will remove the old content of the buildhistory as part of
18# the current bitbake invocation and replace it with information about what was built
19# during the build.
20#
21# This is meant to be used in continuous integration (CI) systems when invoking bitbake
22# for full world builds. The effect in that case is that information about packages
23# that no longer get build also gets removed from the buildhistory, which is not
24# the case otherwise.
25#
26# The advantage over manually cleaning the buildhistory outside of bitbake is that
27# the "version-going-backwards" check still works. When relying on that, be careful
28# about failed world builds: they will lead to incomplete information in the
29# buildhistory because information about packages that could not be built will
30# also get removed. A CI system should handle that by discarding the buildhistory
31# of failed builds.
32#
33# The expected usage is via auto.conf, but passing via the command line also works
34# with: BB_ENV_EXTRAWHITE=BUILDHISTORY_RESET BUILDHISTORY_RESET=1
35BUILDHISTORY_RESET ?= ""
36
37BUILDHISTORY_OLD_DIR = "${BUILDHISTORY_DIR}/${@ "old" if "${BUILDHISTORY_RESET}" else ""}"
38BUILDHISTORY_OLD_DIR_PACKAGE = "${BUILDHISTORY_OLD_DIR}/packages/${MULTIMACH_TARGET_SYS}/${PN}"
39BUILDHISTORY_DIR_SDK = "${BUILDHISTORY_DIR}/sdk/${SDK_NAME}${SDK_EXT}/${IMAGE_BASENAME}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050040BUILDHISTORY_IMAGE_FILES ?= "/etc/passwd /etc/group"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050041BUILDHISTORY_SDK_FILES ?= "conf/local.conf conf/bblayers.conf conf/auto.conf conf/locked-sigs.inc conf/devtool.conf"
Brad Bishop316dfdd2018-06-25 12:45:53 -040042BUILDHISTORY_COMMIT ?= "1"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050043BUILDHISTORY_COMMIT_AUTHOR ?= "buildhistory <buildhistory@${DISTRO}>"
44BUILDHISTORY_PUSH_REPO ?= ""
Andrew Geissler82c905d2020-04-13 13:39:40 -050045BUILDHISTORY_TAG ?= "build"
Andrew Geissler09036742021-06-25 14:25:14 -050046BUILDHISTORY_PATH_PREFIX_STRIP ?= ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -050047
Patrick Williams213cb262021-08-07 19:21:33 -050048SSTATEPOSTINSTFUNCS:append = " buildhistory_emit_pkghistory"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050049# We want to avoid influencing the signatures of sstate tasks - first the function itself:
Patrick Williamsc124f4f2015-09-15 14:41:29 -050050sstate_install[vardepsexclude] += "buildhistory_emit_pkghistory"
51# then the value added to SSTATEPOSTINSTFUNCS:
52SSTATEPOSTINSTFUNCS[vardepvalueexclude] .= "| buildhistory_emit_pkghistory"
53
Brad Bishop6e60e8b2018-02-01 10:27:11 -050054# Similarly for our function that gets the output signatures
Patrick Williams213cb262021-08-07 19:21:33 -050055SSTATEPOSTUNPACKFUNCS:append = " buildhistory_emit_outputsigs"
Brad Bishop6e60e8b2018-02-01 10:27:11 -050056sstate_installpkgdir[vardepsexclude] += "buildhistory_emit_outputsigs"
57SSTATEPOSTUNPACKFUNCS[vardepvalueexclude] .= "| buildhistory_emit_outputsigs"
58
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050059# All items excepts those listed here will be removed from a recipe's
60# build history directory by buildhistory_emit_pkghistory(). This is
61# necessary because some of these items (package directories, files that
62# we no longer emit) might be obsolete.
Patrick Williamsc124f4f2015-09-15 14:41:29 -050063#
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050064# When extending build history, derive your class from buildhistory.bbclass
65# and extend this list here with the additional files created by the derived
66# class.
Brad Bishop96ff1982019-08-19 13:50:42 -040067BUILDHISTORY_PRESERVE = "latest latest_srcrev sysroot"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050068
Patrick Williamsc0f7c042017-02-23 20:41:17 -060069PATCH_GIT_USER_EMAIL ?= "buildhistory@oe"
70PATCH_GIT_USER_NAME ?= "OpenEmbedded"
71
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050072#
Brad Bishop96ff1982019-08-19 13:50:42 -040073# Write out the contents of the sysroot
74#
75buildhistory_emit_sysroot() {
76 mkdir --parents ${BUILDHISTORY_DIR_PACKAGE}
77 case ${CLASSOVERRIDE} in
78 class-native|class-cross|class-crosssdk)
79 BASE=${SYSROOT_DESTDIR}/${STAGING_DIR_NATIVE}
80 ;;
81 *)
82 BASE=${SYSROOT_DESTDIR}
83 ;;
84 esac
85 buildhistory_list_files_no_owners $BASE ${BUILDHISTORY_DIR_PACKAGE}/sysroot
86}
87
88#
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050089# Write out metadata about this package for comparison when writing future packages
Patrick Williamsc124f4f2015-09-15 14:41:29 -050090#
91python buildhistory_emit_pkghistory() {
Brad Bishop96ff1982019-08-19 13:50:42 -040092 if d.getVar('BB_CURRENTTASK') in ['populate_sysroot', 'populate_sysroot_setscene']:
93 bb.build.exec_func("buildhistory_emit_sysroot", d)
94
Brad Bishop6e60e8b2018-02-01 10:27:11 -050095 if not d.getVar('BB_CURRENTTASK') in ['packagedata', 'packagedata_setscene']:
Patrick Williamsc124f4f2015-09-15 14:41:29 -050096 return 0
97
Brad Bishop6e60e8b2018-02-01 10:27:11 -050098 if not "package" in (d.getVar('BUILDHISTORY_FEATURES') or "").split():
Patrick Williamsc124f4f2015-09-15 14:41:29 -050099 return 0
100
101 import re
102 import json
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800103 import shlex
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500104 import errno
105
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500106 pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE')
107 oldpkghistdir = d.getVar('BUILDHISTORY_OLD_DIR_PACKAGE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500108
109 class RecipeInfo:
110 def __init__(self, name):
111 self.name = name
112 self.pe = "0"
113 self.pv = "0"
114 self.pr = "r0"
115 self.depends = ""
116 self.packages = ""
117 self.srcrev = ""
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500118 self.layer = ""
Andrew Geissler6ce62a22020-11-30 19:58:47 -0600119 self.license = ""
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500120 self.config = ""
Andrew Geissler4c19ea12020-10-27 13:52:24 -0500121 self.src_uri = ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500122
123
124 class PackageInfo:
125 def __init__(self, name):
126 self.name = name
127 self.pe = "0"
128 self.pv = "0"
129 self.pr = "r0"
130 # pkg/pkge/pkgv/pkgr should be empty because we want to be able to default them
131 self.pkg = ""
132 self.pkge = ""
133 self.pkgv = ""
134 self.pkgr = ""
135 self.size = 0
136 self.depends = ""
137 self.rprovides = ""
138 self.rdepends = ""
139 self.rrecommends = ""
140 self.rsuggests = ""
141 self.rreplaces = ""
142 self.rconflicts = ""
143 self.files = ""
144 self.filelist = ""
145 # Variables that need to be written to their own separate file
146 self.filevars = dict.fromkeys(['pkg_preinst', 'pkg_postinst', 'pkg_prerm', 'pkg_postrm'])
147
148 # Should check PACKAGES here to see if anything removed
149
150 def readPackageInfo(pkg, histfile):
151 pkginfo = PackageInfo(pkg)
152 with open(histfile, "r") as f:
153 for line in f:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500154 lns = line.split('=', 1)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500155 name = lns[0].strip()
156 value = lns[1].strip(" \t\r\n").strip('"')
157 if name == "PE":
158 pkginfo.pe = value
159 elif name == "PV":
160 pkginfo.pv = value
161 elif name == "PR":
162 pkginfo.pr = value
163 elif name == "PKG":
164 pkginfo.pkg = value
165 elif name == "PKGE":
166 pkginfo.pkge = value
167 elif name == "PKGV":
168 pkginfo.pkgv = value
169 elif name == "PKGR":
170 pkginfo.pkgr = value
171 elif name == "RPROVIDES":
172 pkginfo.rprovides = value
173 elif name == "RDEPENDS":
174 pkginfo.rdepends = value
175 elif name == "RRECOMMENDS":
176 pkginfo.rrecommends = value
177 elif name == "RSUGGESTS":
178 pkginfo.rsuggests = value
179 elif name == "RREPLACES":
180 pkginfo.rreplaces = value
181 elif name == "RCONFLICTS":
182 pkginfo.rconflicts = value
183 elif name == "PKGSIZE":
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600184 pkginfo.size = int(value)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500185 elif name == "FILES":
186 pkginfo.files = value
187 elif name == "FILELIST":
188 pkginfo.filelist = value
189 # Apply defaults
190 if not pkginfo.pkg:
191 pkginfo.pkg = pkginfo.name
192 if not pkginfo.pkge:
193 pkginfo.pkge = pkginfo.pe
194 if not pkginfo.pkgv:
195 pkginfo.pkgv = pkginfo.pv
196 if not pkginfo.pkgr:
197 pkginfo.pkgr = pkginfo.pr
198 return pkginfo
199
200 def getlastpkgversion(pkg):
201 try:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500202 histfile = os.path.join(oldpkghistdir, pkg, "latest")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500203 return readPackageInfo(pkg, histfile)
204 except EnvironmentError:
205 return None
206
207 def sortpkglist(string):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500208 pkgiter = re.finditer(r'[a-zA-Z0-9.+-]+( \([><=]+[^)]+\))?', string, 0)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500209 pkglist = [p.group(0) for p in pkgiter]
210 pkglist.sort()
211 return ' '.join(pkglist)
212
213 def sortlist(string):
214 items = string.split(' ')
215 items.sort()
216 return ' '.join(items)
217
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500218 pn = d.getVar('PN')
219 pe = d.getVar('PE') or "0"
220 pv = d.getVar('PV')
221 pr = d.getVar('PR')
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500222 layer = bb.utils.get_file_layer(d.getVar('FILE'), d)
Andrew Geissler6ce62a22020-11-30 19:58:47 -0600223 license = d.getVar('LICENSE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500224
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500225 pkgdata_dir = d.getVar('PKGDATA_DIR')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500226 packages = ""
227 try:
228 with open(os.path.join(pkgdata_dir, pn)) as f:
229 for line in f.readlines():
230 if line.startswith('PACKAGES: '):
231 packages = oe.utils.squashspaces(line.split(': ', 1)[1])
232 break
233 except IOError as e:
234 if e.errno == errno.ENOENT:
235 # Probably a -cross recipe, just ignore
236 return 0
237 else:
238 raise
239
240 packagelist = packages.split()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500241 preserve = d.getVar('BUILDHISTORY_PRESERVE').split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500242 if not os.path.exists(pkghistdir):
243 bb.utils.mkdirhier(pkghistdir)
244 else:
245 # Remove files for packages that no longer exist
246 for item in os.listdir(pkghistdir):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500247 if item not in preserve:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500248 if item not in packagelist:
249 itempath = os.path.join(pkghistdir, item)
250 if os.path.isdir(itempath):
251 for subfile in os.listdir(itempath):
252 os.unlink(os.path.join(itempath, subfile))
253 os.rmdir(itempath)
254 else:
255 os.unlink(itempath)
256
257 rcpinfo = RecipeInfo(pn)
258 rcpinfo.pe = pe
259 rcpinfo.pv = pv
260 rcpinfo.pr = pr
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500261 rcpinfo.depends = sortlist(oe.utils.squashspaces(d.getVar('DEPENDS') or ""))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500262 rcpinfo.packages = packages
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500263 rcpinfo.layer = layer
Andrew Geissler6ce62a22020-11-30 19:58:47 -0600264 rcpinfo.license = license
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500265 rcpinfo.config = sortlist(oe.utils.squashspaces(d.getVar('PACKAGECONFIG') or ""))
Andrew Geissler4c19ea12020-10-27 13:52:24 -0500266 rcpinfo.src_uri = oe.utils.squashspaces(d.getVar('SRC_URI') or "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500267 write_recipehistory(rcpinfo, d)
268
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500269 bb.build.exec_func("read_subpackage_metadata", d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500270
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500271 for pkg in packagelist:
272 localdata = d.createCopy()
273 localdata.setVar('OVERRIDES', d.getVar("OVERRIDES", False) + ":" + pkg)
274
275 pkge = localdata.getVar("PKGE") or '0'
276 pkgv = localdata.getVar("PKGV")
277 pkgr = localdata.getVar("PKGR")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500278 #
279 # Find out what the last version was
280 # Make sure the version did not decrease
281 #
282 lastversion = getlastpkgversion(pkg)
283 if lastversion:
284 last_pkge = lastversion.pkge
285 last_pkgv = lastversion.pkgv
286 last_pkgr = lastversion.pkgr
287 r = bb.utils.vercmp((pkge, pkgv, pkgr), (last_pkge, last_pkgv, last_pkgr))
288 if r < 0:
Brad Bishop1d80a2e2019-11-15 16:35:03 -0500289 msg = "Package version for package %s went backwards which would break package feeds (from %s:%s-%s to %s:%s-%s)" % (pkg, last_pkge, last_pkgv, last_pkgr, pkge, pkgv, pkgr)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500290 package_qa_handle_error("version-going-backwards", msg, d)
291
292 pkginfo = PackageInfo(pkg)
293 # Apparently the version can be different on a per-package basis (see Python)
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500294 pkginfo.pe = localdata.getVar("PE") or '0'
295 pkginfo.pv = localdata.getVar("PV")
296 pkginfo.pr = localdata.getVar("PR")
297 pkginfo.pkg = localdata.getVar("PKG")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500298 pkginfo.pkge = pkge
299 pkginfo.pkgv = pkgv
300 pkginfo.pkgr = pkgr
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500301 pkginfo.rprovides = sortpkglist(oe.utils.squashspaces(localdata.getVar("RPROVIDES") or ""))
302 pkginfo.rdepends = sortpkglist(oe.utils.squashspaces(localdata.getVar("RDEPENDS") or ""))
303 pkginfo.rrecommends = sortpkglist(oe.utils.squashspaces(localdata.getVar("RRECOMMENDS") or ""))
304 pkginfo.rsuggests = sortpkglist(oe.utils.squashspaces(localdata.getVar("RSUGGESTS") or ""))
305 pkginfo.replaces = sortpkglist(oe.utils.squashspaces(localdata.getVar("RREPLACES") or ""))
306 pkginfo.rconflicts = sortpkglist(oe.utils.squashspaces(localdata.getVar("RCONFLICTS") or ""))
307 pkginfo.files = oe.utils.squashspaces(localdata.getVar("FILES") or "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500308 for filevar in pkginfo.filevars:
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500309 pkginfo.filevars[filevar] = localdata.getVar(filevar) or ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500310
311 # Gather information about packaged files
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500312 val = localdata.getVar('FILES_INFO') or ''
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500313 dictval = json.loads(val)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600314 filelist = list(dictval.keys())
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500315 filelist.sort()
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800316 pkginfo.filelist = " ".join([shlex.quote(x) for x in filelist])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500317
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500318 pkginfo.size = int(localdata.getVar('PKGSIZE') or '0')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500319
320 write_pkghistory(pkginfo, d)
321
322 # Create files-in-<package-name>.txt files containing a list of files of each recipe's package
323 bb.build.exec_func("buildhistory_list_pkg_files", d)
324}
325
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500326python buildhistory_emit_outputsigs() {
327 if not "task" in (d.getVar('BUILDHISTORY_FEATURES') or "").split():
328 return
329
330 import hashlib
331
332 taskoutdir = os.path.join(d.getVar('BUILDHISTORY_DIR'), 'task', 'output')
333 bb.utils.mkdirhier(taskoutdir)
334 currenttask = d.getVar('BB_CURRENTTASK')
335 pn = d.getVar('PN')
336 taskfile = os.path.join(taskoutdir, '%s.%s' % (pn, currenttask))
337
338 cwd = os.getcwd()
339 filesigs = {}
340 for root, _, files in os.walk(cwd):
341 for fname in files:
342 if fname == 'fixmepath':
343 continue
344 fullpath = os.path.join(root, fname)
345 try:
346 if os.path.islink(fullpath):
347 sha256 = hashlib.sha256(os.readlink(fullpath).encode('utf-8')).hexdigest()
348 elif os.path.isfile(fullpath):
349 sha256 = bb.utils.sha256_file(fullpath)
350 else:
351 continue
352 except OSError:
353 bb.warn('buildhistory: unable to read %s to get output signature' % fullpath)
354 continue
355 filesigs[os.path.relpath(fullpath, cwd)] = sha256
356 with open(taskfile, 'w') as f:
357 for fpath, fsig in sorted(filesigs.items(), key=lambda item: item[0]):
358 f.write('%s %s\n' % (fpath, fsig))
359}
360
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500361
362def write_recipehistory(rcpinfo, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500363 bb.debug(2, "Writing recipe history")
364
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500365 pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500366
367 infofile = os.path.join(pkghistdir, "latest")
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600368 with open(infofile, "w") as f:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500369 if rcpinfo.pe != "0":
370 f.write(u"PE = %s\n" % rcpinfo.pe)
371 f.write(u"PV = %s\n" % rcpinfo.pv)
372 f.write(u"PR = %s\n" % rcpinfo.pr)
373 f.write(u"DEPENDS = %s\n" % rcpinfo.depends)
374 f.write(u"PACKAGES = %s\n" % rcpinfo.packages)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500375 f.write(u"LAYER = %s\n" % rcpinfo.layer)
Andrew Geissler6ce62a22020-11-30 19:58:47 -0600376 f.write(u"LICENSE = %s\n" % rcpinfo.license)
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500377 f.write(u"CONFIG = %s\n" % rcpinfo.config)
Andrew Geissler4c19ea12020-10-27 13:52:24 -0500378 f.write(u"SRC_URI = %s\n" % rcpinfo.src_uri)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500379
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500380 write_latest_srcrev(d, pkghistdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500381
382def write_pkghistory(pkginfo, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500383 bb.debug(2, "Writing package history for package %s" % pkginfo.name)
384
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500385 pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500386
387 pkgpath = os.path.join(pkghistdir, pkginfo.name)
388 if not os.path.exists(pkgpath):
389 bb.utils.mkdirhier(pkgpath)
390
391 infofile = os.path.join(pkgpath, "latest")
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600392 with open(infofile, "w") as f:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500393 if pkginfo.pe != "0":
394 f.write(u"PE = %s\n" % pkginfo.pe)
395 f.write(u"PV = %s\n" % pkginfo.pv)
396 f.write(u"PR = %s\n" % pkginfo.pr)
397
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600398 if pkginfo.pkg != pkginfo.name:
399 f.write(u"PKG = %s\n" % pkginfo.pkg)
400 if pkginfo.pkge != pkginfo.pe:
401 f.write(u"PKGE = %s\n" % pkginfo.pkge)
402 if pkginfo.pkgv != pkginfo.pv:
403 f.write(u"PKGV = %s\n" % pkginfo.pkgv)
404 if pkginfo.pkgr != pkginfo.pr:
405 f.write(u"PKGR = %s\n" % pkginfo.pkgr)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500406 f.write(u"RPROVIDES = %s\n" % pkginfo.rprovides)
407 f.write(u"RDEPENDS = %s\n" % pkginfo.rdepends)
408 f.write(u"RRECOMMENDS = %s\n" % pkginfo.rrecommends)
409 if pkginfo.rsuggests:
410 f.write(u"RSUGGESTS = %s\n" % pkginfo.rsuggests)
411 if pkginfo.rreplaces:
412 f.write(u"RREPLACES = %s\n" % pkginfo.rreplaces)
413 if pkginfo.rconflicts:
414 f.write(u"RCONFLICTS = %s\n" % pkginfo.rconflicts)
415 f.write(u"PKGSIZE = %d\n" % pkginfo.size)
416 f.write(u"FILES = %s\n" % pkginfo.files)
417 f.write(u"FILELIST = %s\n" % pkginfo.filelist)
418
419 for filevar in pkginfo.filevars:
420 filevarpath = os.path.join(pkgpath, "latest.%s" % filevar)
421 val = pkginfo.filevars[filevar]
422 if val:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600423 with open(filevarpath, "w") as f:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500424 f.write(val)
425 else:
426 if os.path.exists(filevarpath):
427 os.unlink(filevarpath)
428
429#
430# rootfs_type can be: image, sdk_target, sdk_host
431#
432def buildhistory_list_installed(d, rootfs_type="image"):
433 from oe.rootfs import image_list_installed_packages
434 from oe.sdk import sdk_list_installed_packages
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500435 from oe.utils import format_pkg_list
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500436
Andrew Geissler635e0e42020-08-21 15:58:33 -0500437 process_list = [('file', 'bh_installed_pkgs_%s.txt' % os.getpid()),\
438 ('deps', 'bh_installed_pkgs_deps_%s.txt' % os.getpid())]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500439
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500440 if rootfs_type == "image":
441 pkgs = image_list_installed_packages(d)
442 else:
443 pkgs = sdk_list_installed_packages(d, rootfs_type == "sdk_target")
444
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500445 for output_type, output_file in process_list:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500446 output_file_full = os.path.join(d.getVar('WORKDIR'), output_file)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500447
448 with open(output_file_full, 'w') as output:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500449 output.write(format_pkg_list(pkgs, output_type))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500450
451python buildhistory_list_installed_image() {
452 buildhistory_list_installed(d)
453}
454
455python buildhistory_list_installed_sdk_target() {
456 buildhistory_list_installed(d, "sdk_target")
457}
458
459python buildhistory_list_installed_sdk_host() {
460 buildhistory_list_installed(d, "sdk_host")
461}
462
463buildhistory_get_installed() {
464 mkdir -p $1
465
466 # Get list of installed packages
467 pkgcache="$1/installed-packages.tmp"
Andrew Geissler635e0e42020-08-21 15:58:33 -0500468 cat ${WORKDIR}/bh_installed_pkgs_${PID}.txt | sort > $pkgcache && rm ${WORKDIR}/bh_installed_pkgs_${PID}.txt
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500469
470 cat $pkgcache | awk '{ print $1 }' > $1/installed-package-names.txt
Andrew Geissler635e0e42020-08-21 15:58:33 -0500471
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500472 if [ -s $pkgcache ] ; then
473 cat $pkgcache | awk '{ print $2 }' | xargs -n1 basename > $1/installed-packages.txt
474 else
475 printf "" > $1/installed-packages.txt
476 fi
477
478 # Produce dependency graph
479 # First, quote each name to handle characters that cause issues for dot
Andrew Geissler635e0e42020-08-21 15:58:33 -0500480 sed 's:\([^| ]*\):"\1":g' ${WORKDIR}/bh_installed_pkgs_deps_${PID}.txt > $1/depends.tmp &&
481 rm ${WORKDIR}/bh_installed_pkgs_deps_${PID}.txt
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500482 # Remove lines with rpmlib(...) and config(...) dependencies, change the
483 # delimiter from pipe to "->", set the style for recommend lines and
484 # turn versioned dependencies into edge labels.
485 sed -i -e '/rpmlib(/d' \
486 -e '/config(/d' \
487 -e 's:|: -> :' \
488 -e 's:"\[REC\]":[style=dotted]:' \
489 -e 's:"\([<>=]\+\)" "\([^"]*\)":[label="\1 \2"]:' \
490 $1/depends.tmp
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500491 # Add header, sorted and de-duped contents and footer and then delete the temp file
492 printf "digraph depends {\n node [shape=plaintext]\n" > $1/depends.dot
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500493 cat $1/depends.tmp | sort -u >> $1/depends.dot
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500494 echo "}" >> $1/depends.dot
495 rm $1/depends.tmp
496
497 # Produce installed package sizes list
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500498 oe-pkgdata-util -p ${PKGDATA_DIR} read-value "PKGSIZE" -n -f $pkgcache > $1/installed-package-sizes.tmp
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500499 cat $1/installed-package-sizes.tmp | awk '{print $2 "\tKiB\t" $1}' | sort -n -r > $1/installed-package-sizes.txt
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500500 rm $1/installed-package-sizes.tmp
501
502 # We're now done with the cache, delete it
503 rm $pkgcache
504
505 if [ "$2" != "sdk" ] ; then
506 # Produce some cut-down graphs (for readability)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500507 grep -v kernel-image $1/depends.dot | grep -v kernel-3 | grep -v kernel-4 > $1/depends-nokernel.dot
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500508 grep -v libc6 $1/depends-nokernel.dot | grep -v libgcc > $1/depends-nokernel-nolibc.dot
509 grep -v update- $1/depends-nokernel-nolibc.dot > $1/depends-nokernel-nolibc-noupdate.dot
510 grep -v kernel-module $1/depends-nokernel-nolibc-noupdate.dot > $1/depends-nokernel-nolibc-noupdate-nomodules.dot
511 fi
512
513 # add complementary package information
514 if [ -e ${WORKDIR}/complementary_pkgs.txt ]; then
515 cp ${WORKDIR}/complementary_pkgs.txt $1
516 fi
517}
518
519buildhistory_get_image_installed() {
520 # Anything requiring the use of the packaging system should be done in here
521 # in case the packaging files are going to be removed for this image
522
523 if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'image', '1', '0', d)}" = "0" ] ; then
524 return
525 fi
526
527 buildhistory_get_installed ${BUILDHISTORY_DIR_IMAGE}
528}
529
530buildhistory_get_sdk_installed() {
531 # Anything requiring the use of the packaging system should be done in here
532 # in case the packaging files are going to be removed for this SDK
533
534 if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'sdk', '1', '0', d)}" = "0" ] ; then
535 return
536 fi
537
538 buildhistory_get_installed ${BUILDHISTORY_DIR_SDK}/$1 sdk
539}
540
541buildhistory_get_sdk_installed_host() {
542 buildhistory_get_sdk_installed host
543}
544
545buildhistory_get_sdk_installed_target() {
546 buildhistory_get_sdk_installed target
547}
548
549buildhistory_list_files() {
550 # List the files in the specified directory, but exclude date/time etc.
Brad Bishop19323692019-04-05 15:28:33 -0400551 # This is somewhat messy, but handles where the size is not printed for device files under pseudo
552 ( cd $1
553 find_cmd='find . ! -path . -printf "%M %-10u %-10g %10s %p -> %l\n"'
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500554 if [ "$3" = "fakeroot" ] ; then
Brad Bishop19323692019-04-05 15:28:33 -0400555 eval ${FAKEROOTENV} ${FAKEROOTCMD} $find_cmd
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500556 else
Brad Bishop19323692019-04-05 15:28:33 -0400557 eval $find_cmd
558 fi | sort -k5 | sed 's/ * -> $//' > $2 )
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500559}
560
Brad Bishop96ff1982019-08-19 13:50:42 -0400561buildhistory_list_files_no_owners() {
562 # List the files in the specified directory, but exclude date/time etc.
563 # Also don't output the ownership data, but instead output just - - so
564 # that the same parsing code as for _list_files works.
565 # This is somewhat messy, but handles where the size is not printed for device files under pseudo
566 ( cd $1
567 find_cmd='find . ! -path . -printf "%M - - %10s %p -> %l\n"'
568 if [ "$3" = "fakeroot" ] ; then
569 eval ${FAKEROOTENV} ${FAKEROOTCMD} "$find_cmd"
570 else
571 eval "$find_cmd"
572 fi | sort -k5 | sed 's/ * -> $//' > $2 )
573}
574
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500575buildhistory_list_pkg_files() {
576 # Create individual files-in-package for each recipe's package
577 for pkgdir in $(find ${PKGDEST}/* -maxdepth 0 -type d); do
578 pkgname=$(basename $pkgdir)
579 outfolder="${BUILDHISTORY_DIR_PACKAGE}/$pkgname"
580 outfile="$outfolder/files-in-package.txt"
581 # Make sure the output folder exists so we can create the file
582 if [ ! -d $outfolder ] ; then
583 bbdebug 2 "Folder $outfolder does not exist, file $outfile not created"
584 continue
585 fi
586 buildhistory_list_files $pkgdir $outfile fakeroot
587 done
588}
589
590buildhistory_get_imageinfo() {
591 if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'image', '1', '0', d)}" = "0" ] ; then
592 return
593 fi
594
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500595 mkdir -p ${BUILDHISTORY_DIR_IMAGE}
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500596 buildhistory_list_files ${IMAGE_ROOTFS} ${BUILDHISTORY_DIR_IMAGE}/files-in-image.txt
597
598 # Collect files requested in BUILDHISTORY_IMAGE_FILES
599 rm -rf ${BUILDHISTORY_DIR_IMAGE}/image-files
600 for f in ${BUILDHISTORY_IMAGE_FILES}; do
601 if [ -f ${IMAGE_ROOTFS}/$f ] ; then
602 mkdir -p ${BUILDHISTORY_DIR_IMAGE}/image-files/`dirname $f`
603 cp ${IMAGE_ROOTFS}/$f ${BUILDHISTORY_DIR_IMAGE}/image-files/$f
604 fi
605 done
606
607 # Record some machine-readable meta-information about the image
608 printf "" > ${BUILDHISTORY_DIR_IMAGE}/image-info.txt
609 cat >> ${BUILDHISTORY_DIR_IMAGE}/image-info.txt <<END
610${@buildhistory_get_imagevars(d)}
611END
612 imagesize=`du -ks ${IMAGE_ROOTFS} | awk '{ print $1 }'`
613 echo "IMAGESIZE = $imagesize" >> ${BUILDHISTORY_DIR_IMAGE}/image-info.txt
614
615 # Add some configuration information
616 echo "${MACHINE}: ${IMAGE_BASENAME} configured for ${DISTRO} ${DISTRO_VERSION}" > ${BUILDHISTORY_DIR_IMAGE}/build-id.txt
617
618 cat >> ${BUILDHISTORY_DIR_IMAGE}/build-id.txt <<END
619${@buildhistory_get_build_id(d)}
620END
621}
622
623buildhistory_get_sdkinfo() {
624 if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'sdk', '1', '0', d)}" = "0" ] ; then
625 return
626 fi
627
628 buildhistory_list_files ${SDK_OUTPUT} ${BUILDHISTORY_DIR_SDK}/files-in-sdk.txt
629
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500630 # Collect files requested in BUILDHISTORY_SDK_FILES
631 rm -rf ${BUILDHISTORY_DIR_SDK}/sdk-files
632 for f in ${BUILDHISTORY_SDK_FILES}; do
633 if [ -f ${SDK_OUTPUT}/${SDKPATH}/$f ] ; then
634 mkdir -p ${BUILDHISTORY_DIR_SDK}/sdk-files/`dirname $f`
635 cp ${SDK_OUTPUT}/${SDKPATH}/$f ${BUILDHISTORY_DIR_SDK}/sdk-files/$f
636 fi
637 done
638
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500639 # Record some machine-readable meta-information about the SDK
640 printf "" > ${BUILDHISTORY_DIR_SDK}/sdk-info.txt
641 cat >> ${BUILDHISTORY_DIR_SDK}/sdk-info.txt <<END
642${@buildhistory_get_sdkvars(d)}
643END
644 sdksize=`du -ks ${SDK_OUTPUT} | awk '{ print $1 }'`
645 echo "SDKSIZE = $sdksize" >> ${BUILDHISTORY_DIR_SDK}/sdk-info.txt
646}
647
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500648python buildhistory_get_extra_sdkinfo() {
649 import operator
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500650 from oe.sdk import get_extra_sdkinfo
651
652 sstate_dir = d.expand('${SDK_OUTPUT}/${SDKPATH}/sstate-cache')
653 extra_info = get_extra_sdkinfo(sstate_dir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500654
655 if d.getVar('BB_CURRENTTASK') == 'populate_sdk_ext' and \
656 "sdk" in (d.getVar('BUILDHISTORY_FEATURES') or "").split():
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500657 with open(d.expand('${BUILDHISTORY_DIR_SDK}/sstate-package-sizes.txt'), 'w') as f:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500658 filesizes_sorted = sorted(extra_info['filesizes'].items(), key=operator.itemgetter(1, 0), reverse=True)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500659 for fn, size in filesizes_sorted:
660 f.write('%10d KiB %s\n' % (size, fn))
661 with open(d.expand('${BUILDHISTORY_DIR_SDK}/sstate-task-sizes.txt'), 'w') as f:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500662 tasksizes_sorted = sorted(extra_info['tasksizes'].items(), key=operator.itemgetter(1, 0), reverse=True)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500663 for task, size in tasksizes_sorted:
664 f.write('%10d KiB %s\n' % (size, task))
665}
666
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500667# By using ROOTFS_POSTUNINSTALL_COMMAND we get in after uninstallation of
668# unneeded packages but before the removal of packaging files
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500669ROOTFS_POSTUNINSTALL_COMMAND += "buildhistory_list_installed_image ;"
670ROOTFS_POSTUNINSTALL_COMMAND += "buildhistory_get_image_installed ;"
671ROOTFS_POSTUNINSTALL_COMMAND[vardepvalueexclude] .= "| buildhistory_list_installed_image ;| buildhistory_get_image_installed ;"
672ROOTFS_POSTUNINSTALL_COMMAND[vardepsexclude] += "buildhistory_list_installed_image buildhistory_get_image_installed"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500673
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500674IMAGE_POSTPROCESS_COMMAND += "buildhistory_get_imageinfo ;"
675IMAGE_POSTPROCESS_COMMAND[vardepvalueexclude] .= "| buildhistory_get_imageinfo ;"
676IMAGE_POSTPROCESS_COMMAND[vardepsexclude] += "buildhistory_get_imageinfo"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500677
678# We want these to be the last run so that we get called after complementary package installation
Patrick Williams213cb262021-08-07 19:21:33 -0500679POPULATE_SDK_POST_TARGET_COMMAND:append = " buildhistory_list_installed_sdk_target;"
680POPULATE_SDK_POST_TARGET_COMMAND:append = " buildhistory_get_sdk_installed_target;"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500681POPULATE_SDK_POST_TARGET_COMMAND[vardepvalueexclude] .= "| buildhistory_list_installed_sdk_target;| buildhistory_get_sdk_installed_target;"
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500682POPULATE_SDK_POST_TARGET_COMMAND[vardepsexclude] += "buildhistory_list_installed_sdk_target buildhistory_get_sdk_installed_target"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500683
Patrick Williams213cb262021-08-07 19:21:33 -0500684POPULATE_SDK_POST_HOST_COMMAND:append = " buildhistory_list_installed_sdk_host;"
685POPULATE_SDK_POST_HOST_COMMAND:append = " buildhistory_get_sdk_installed_host;"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500686POPULATE_SDK_POST_HOST_COMMAND[vardepvalueexclude] .= "| buildhistory_list_installed_sdk_host;| buildhistory_get_sdk_installed_host;"
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500687POPULATE_SDK_POST_HOST_COMMAND[vardepsexclude] += "buildhistory_list_installed_sdk_host buildhistory_get_sdk_installed_host"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500688
Patrick Williams213cb262021-08-07 19:21:33 -0500689SDK_POSTPROCESS_COMMAND:append = " buildhistory_get_sdkinfo ; buildhistory_get_extra_sdkinfo; "
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500690SDK_POSTPROCESS_COMMAND[vardepvalueexclude] .= "| buildhistory_get_sdkinfo ; buildhistory_get_extra_sdkinfo; "
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500691SDK_POSTPROCESS_COMMAND[vardepsexclude] += "buildhistory_get_sdkinfo buildhistory_get_extra_sdkinfo"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500692
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500693python buildhistory_write_sigs() {
694 if not "task" in (d.getVar('BUILDHISTORY_FEATURES') or "").split():
695 return
696
697 # Create sigs file
698 if hasattr(bb.parse.siggen, 'dump_siglist'):
699 taskoutdir = os.path.join(d.getVar('BUILDHISTORY_DIR'), 'task')
700 bb.utils.mkdirhier(taskoutdir)
Andrew Geissler09036742021-06-25 14:25:14 -0500701 bb.parse.siggen.dump_siglist(os.path.join(taskoutdir, 'tasksigs.txt'), d.getVar("BUILDHISTORY_PATH_PREFIX_STRIP"))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500702}
703
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500704def buildhistory_get_build_id(d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500705 if d.getVar('BB_WORKERCONTEXT') != '1':
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500706 return ""
707 localdata = bb.data.createCopy(d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500708 statuslines = []
709 for func in oe.data.typed_value('BUILDCFG_FUNCS', localdata):
710 g = globals()
711 if func not in g:
712 bb.warn("Build configuration function '%s' does not exist" % func)
713 else:
714 flines = g[func](localdata)
715 if flines:
716 statuslines.extend(flines)
717
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500718 statusheader = d.getVar('BUILDCFG_HEADER')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500719 return('\n%s\n%s\n' % (statusheader, '\n'.join(statuslines)))
720
Brad Bishop19323692019-04-05 15:28:33 -0400721def buildhistory_get_modified(path):
722 # copied from get_layer_git_status() in image-buildinfo.bbclass
723 import subprocess
724 try:
725 subprocess.check_output("""cd %s; export PSEUDO_UNLOAD=1; set -e;
726 git diff --quiet --no-ext-diff
727 git diff --quiet --no-ext-diff --cached""" % path,
728 shell=True,
729 stderr=subprocess.STDOUT)
730 return ""
731 except subprocess.CalledProcessError as ex:
732 # Silently treat errors as "modified", without checking for the
733 # (expected) return code 1 in a modified git repo. For example, we get
734 # output and a 129 return code when a layer isn't a git repo at all.
735 return " -- modified"
736
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500737def buildhistory_get_metadata_revs(d):
738 # We want an easily machine-readable format here, so get_layers_branch_rev isn't quite what we want
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500739 layers = (d.getVar("BBLAYERS") or "").split()
Brad Bishop19323692019-04-05 15:28:33 -0400740 medadata_revs = ["%-17s = %s:%s%s" % (os.path.basename(i), \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500741 base_get_metadata_git_branch(i, None).strip(), \
Brad Bishop19323692019-04-05 15:28:33 -0400742 base_get_metadata_git_revision(i, None), \
743 buildhistory_get_modified(i)) \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500744 for i in layers]
745 return '\n'.join(medadata_revs)
746
747def outputvars(vars, listvars, d):
748 vars = vars.split()
749 listvars = listvars.split()
750 ret = ""
751 for var in vars:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500752 value = d.getVar(var) or ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500753 if var in listvars:
754 # Squash out spaces
755 value = oe.utils.squashspaces(value)
756 ret += "%s = %s\n" % (var, value)
757 return ret.rstrip('\n')
758
759def buildhistory_get_imagevars(d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500760 if d.getVar('BB_WORKERCONTEXT') != '1':
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500761 return ""
762 imagevars = "DISTRO DISTRO_VERSION USER_CLASSES IMAGE_CLASSES IMAGE_FEATURES IMAGE_LINGUAS IMAGE_INSTALL BAD_RECOMMENDATIONS NO_RECOMMENDATIONS PACKAGE_EXCLUDE ROOTFS_POSTPROCESS_COMMAND IMAGE_POSTPROCESS_COMMAND"
763 listvars = "USER_CLASSES IMAGE_CLASSES IMAGE_FEATURES IMAGE_LINGUAS IMAGE_INSTALL BAD_RECOMMENDATIONS PACKAGE_EXCLUDE"
764 return outputvars(imagevars, listvars, d)
765
766def buildhistory_get_sdkvars(d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500767 if d.getVar('BB_WORKERCONTEXT') != '1':
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500768 return ""
769 sdkvars = "DISTRO DISTRO_VERSION SDK_NAME SDK_VERSION SDKMACHINE SDKIMAGE_FEATURES BAD_RECOMMENDATIONS NO_RECOMMENDATIONS PACKAGE_EXCLUDE"
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500770 if d.getVar('BB_CURRENTTASK') == 'populate_sdk_ext':
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500771 # Extensible SDK uses some additional variables
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600772 sdkvars += " SDK_LOCAL_CONF_WHITELIST SDK_LOCAL_CONF_BLACKLIST SDK_INHERIT_BLACKLIST SDK_UPDATE_URL SDK_EXT_TYPE SDK_RECRDEP_TASKS SDK_INCLUDE_PKGDATA SDK_INCLUDE_TOOLCHAIN"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500773 listvars = "SDKIMAGE_FEATURES BAD_RECOMMENDATIONS PACKAGE_EXCLUDE SDK_LOCAL_CONF_WHITELIST SDK_LOCAL_CONF_BLACKLIST SDK_INHERIT_BLACKLIST"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500774 return outputvars(sdkvars, listvars, d)
775
776
777def buildhistory_get_cmdline(d):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500778 argv = d.getVar('BB_CMDLINE', False)
779 if argv:
780 if argv[0].endswith('bin/bitbake'):
781 bincmd = 'bitbake'
782 else:
783 bincmd = argv[0]
784 return '%s %s' % (bincmd, ' '.join(argv[1:]))
785 return ''
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500786
787
788buildhistory_single_commit() {
789 if [ "$3" = "" ] ; then
790 commitopts="${BUILDHISTORY_DIR}/ --allow-empty"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500791 shortlogprefix="No changes: "
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500792 else
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500793 commitopts=""
794 shortlogprefix=""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500795 fi
796 if [ "${BUILDHISTORY_BUILD_FAILURES}" = "0" ] ; then
797 result="succeeded"
798 else
799 result="failed"
800 fi
801 case ${BUILDHISTORY_BUILD_INTERRUPTED} in
802 1)
803 result="$result (interrupted)"
804 ;;
805 2)
806 result="$result (force interrupted)"
807 ;;
808 esac
809 commitmsgfile=`mktemp`
810 cat > $commitmsgfile << END
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500811${shortlogprefix}Build ${BUILDNAME} of ${DISTRO} ${DISTRO_VERSION} for machine ${MACHINE} on $2
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500812
813cmd: $1
814
815result: $result
816
817metadata revisions:
818END
819 cat ${BUILDHISTORY_DIR}/metadata-revs >> $commitmsgfile
820 git commit $commitopts -F $commitmsgfile --author "${BUILDHISTORY_COMMIT_AUTHOR}" > /dev/null
821 rm $commitmsgfile
822}
823
824buildhistory_commit() {
825 if [ ! -d ${BUILDHISTORY_DIR} ] ; then
826 # Code above that creates this dir never executed, so there can't be anything to commit
827 return
828 fi
829
830 # Create a machine-readable list of metadata revisions for each layer
831 cat > ${BUILDHISTORY_DIR}/metadata-revs <<END
832${@buildhistory_get_metadata_revs(d)}
833END
834
835 ( cd ${BUILDHISTORY_DIR}/
836 # Initialise the repo if necessary
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500837 if [ ! -e .git ] ; then
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500838 git init -q
839 else
Andrew Geissler82c905d2020-04-13 13:39:40 -0500840 git tag -f ${BUILDHISTORY_TAG}-minus-3 ${BUILDHISTORY_TAG}-minus-2 > /dev/null 2>&1 || true
841 git tag -f ${BUILDHISTORY_TAG}-minus-2 ${BUILDHISTORY_TAG}-minus-1 > /dev/null 2>&1 || true
842 git tag -f ${BUILDHISTORY_TAG}-minus-1 > /dev/null 2>&1 || true
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500843 fi
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600844
845 check_git_config
846
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500847 # Check if there are new/changed files to commit (other than metadata-revs)
848 repostatus=`git status --porcelain | grep -v " metadata-revs$"`
849 HOSTNAME=`hostname 2>/dev/null || echo unknown`
850 CMDLINE="${@buildhistory_get_cmdline(d)}"
851 if [ "$repostatus" != "" ] ; then
852 git add -A .
853 # porcelain output looks like "?? packages/foo/bar"
854 # Ensure we commit metadata-revs with the first commit
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500855 buildhistory_single_commit "$CMDLINE" "$HOSTNAME" dummy
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500856 git gc --auto --quiet
857 else
858 buildhistory_single_commit "$CMDLINE" "$HOSTNAME"
859 fi
860 if [ "${BUILDHISTORY_PUSH_REPO}" != "" ] ; then
861 git push -q ${BUILDHISTORY_PUSH_REPO}
862 fi) || true
863}
864
865python buildhistory_eventhandler() {
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600866 if (e.data.getVar('BUILDHISTORY_FEATURES') or "").strip():
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500867 reset = e.data.getVar("BUILDHISTORY_RESET")
868 olddir = e.data.getVar("BUILDHISTORY_OLD_DIR")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500869 if isinstance(e, bb.event.BuildStarted):
870 if reset:
871 import shutil
872 # Clean up after potentially interrupted build.
873 if os.path.isdir(olddir):
874 shutil.rmtree(olddir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500875 rootdir = e.data.getVar("BUILDHISTORY_DIR")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500876 entries = [ x for x in os.listdir(rootdir) if not x.startswith('.') ]
877 bb.utils.mkdirhier(olddir)
878 for entry in entries:
Andrew Geisslerc926e172021-05-07 16:11:35 -0500879 bb.utils.rename(os.path.join(rootdir, entry),
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500880 os.path.join(olddir, entry))
881 elif isinstance(e, bb.event.BuildCompleted):
882 if reset:
883 import shutil
884 shutil.rmtree(olddir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500885 if e.data.getVar("BUILDHISTORY_COMMIT") == "1":
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500886 bb.note("Writing buildhistory")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500887 bb.build.exec_func("buildhistory_write_sigs", d)
Brad Bishopf3fd2882019-06-21 08:06:37 -0400888 import time
889 start=time.time()
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500890 localdata = bb.data.createCopy(e.data)
891 localdata.setVar('BUILDHISTORY_BUILD_FAILURES', str(e._failures))
892 interrupted = getattr(e, '_interrupted', 0)
893 localdata.setVar('BUILDHISTORY_BUILD_INTERRUPTED', str(interrupted))
894 bb.build.exec_func("buildhistory_commit", localdata)
Brad Bishopf3fd2882019-06-21 08:06:37 -0400895 stop=time.time()
896 bb.note("Writing buildhistory took: %s seconds" % round(stop-start))
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500897 else:
898 bb.note("No commit since BUILDHISTORY_COMMIT != '1'")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500899}
900
901addhandler buildhistory_eventhandler
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500902buildhistory_eventhandler[eventmask] = "bb.event.BuildCompleted bb.event.BuildStarted"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500903
904
905# FIXME this ought to be moved into the fetcher
906def _get_srcrev_values(d):
907 """
908 Return the version strings for the current recipe
909 """
910
911 scms = []
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500912 fetcher = bb.fetch.Fetch(d.getVar('SRC_URI').split(), d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500913 urldata = fetcher.ud
914 for u in urldata:
915 if urldata[u].method.supports_srcrev():
916 scms.append(u)
917
918 autoinc_templ = 'AUTOINC+'
919 dict_srcrevs = {}
920 dict_tag_srcrevs = {}
921 for scm in scms:
922 ud = urldata[scm]
923 for name in ud.names:
924 try:
925 rev = ud.method.sortable_revision(ud, d, name)
926 except TypeError:
927 # support old bitbake versions
928 rev = ud.method.sortable_revision(scm, ud, d, name)
929 # Clean this up when we next bump bitbake version
930 if type(rev) != str:
931 autoinc, rev = rev
932 elif rev.startswith(autoinc_templ):
933 rev = rev[len(autoinc_templ):]
934 dict_srcrevs[name] = rev
935 if 'tag' in ud.parm:
936 tag = ud.parm['tag'];
937 key = name+'_'+tag
938 dict_tag_srcrevs[key] = rev
939 return (dict_srcrevs, dict_tag_srcrevs)
940
941do_fetch[postfuncs] += "write_srcrev"
942do_fetch[vardepsexclude] += "write_srcrev"
943python write_srcrev() {
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500944 write_latest_srcrev(d, d.getVar('BUILDHISTORY_DIR_PACKAGE'))
945}
946
947def write_latest_srcrev(d, pkghistdir):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500948 srcrevfile = os.path.join(pkghistdir, 'latest_srcrev')
949
950 srcrevs, tag_srcrevs = _get_srcrev_values(d)
951 if srcrevs:
952 if not os.path.exists(pkghistdir):
953 bb.utils.mkdirhier(pkghistdir)
954 old_tag_srcrevs = {}
955 if os.path.exists(srcrevfile):
956 with open(srcrevfile) as f:
957 for line in f:
958 if line.startswith('# tag_'):
959 key, value = line.split("=", 1)
960 key = key.replace('# tag_', '').strip()
961 value = value.replace('"', '').strip()
962 old_tag_srcrevs[key] = value
963 with open(srcrevfile, 'w') as f:
964 orig_srcrev = d.getVar('SRCREV', False) or 'INVALID'
965 if orig_srcrev != 'INVALID':
966 f.write('# SRCREV = "%s"\n' % orig_srcrev)
967 if len(srcrevs) > 1:
Brad Bishop19323692019-04-05 15:28:33 -0400968 for name, srcrev in sorted(srcrevs.items()):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500969 orig_srcrev = d.getVar('SRCREV_%s' % name, False)
970 if orig_srcrev:
971 f.write('# SRCREV_%s = "%s"\n' % (name, orig_srcrev))
972 f.write('SRCREV_%s = "%s"\n' % (name, srcrev))
973 else:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500974 f.write('SRCREV = "%s"\n' % next(iter(srcrevs.values())))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500975 if len(tag_srcrevs) > 0:
Brad Bishop19323692019-04-05 15:28:33 -0400976 for name, srcrev in sorted(tag_srcrevs.items()):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500977 f.write('# tag_%s = "%s"\n' % (name, srcrev))
978 if name in old_tag_srcrevs and old_tag_srcrevs[name] != srcrev:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500979 pkg = d.getVar('PN')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500980 bb.warn("Revision for tag %s in package %s was changed since last build (from %s to %s)" % (name, pkg, old_tag_srcrevs[name], srcrev))
981
982 else:
983 if os.path.exists(srcrevfile):
984 os.remove(srcrevfile)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500985
986do_testimage[postfuncs] += "write_ptest_result"
987do_testimage[vardepsexclude] += "write_ptest_result"
988
989python write_ptest_result() {
990 write_latest_ptest_result(d, d.getVar('BUILDHISTORY_DIR'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500991}
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500992
993def write_latest_ptest_result(d, histdir):
994 import glob
995 import subprocess
996 test_log_dir = d.getVar('TEST_LOG_DIR')
997 input_ptest = os.path.join(test_log_dir, 'ptest_log')
998 output_ptest = os.path.join(histdir, 'ptest')
999 if os.path.exists(input_ptest):
1000 try:
1001 # Lock it avoid race issue
1002 lock = bb.utils.lockfile(output_ptest + "/ptest.lock")
1003 bb.utils.mkdirhier(output_ptest)
1004 oe.path.copytree(input_ptest, output_ptest)
1005 # Sort test result
1006 for result in glob.glob('%s/pass.fail.*' % output_ptest):
1007 bb.debug(1, 'Processing %s' % result)
1008 cmd = ['sort', result, '-o', result]
1009 bb.debug(1, 'Running %s' % cmd)
1010 ret = subprocess.call(cmd)
1011 if ret != 0:
1012 bb.error('Failed to run %s!' % cmd)
1013 finally:
1014 bb.utils.unlockfile(lock)