blob: f986f7c794506ed9876968e740fda83576e7bf15 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001#
2# Records history of build output in order to detect regressions
3#
4# Based in part on testlab.bbclass and packagehistory.bbclass
5#
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05006# Copyright (C) 2011-2016 Intel Corporation
Patrick Williamsc124f4f2015-09-15 14:41:29 -05007# Copyright (C) 2007-2011 Koen Kooi <koen@openembedded.org>
8#
9
10BUILDHISTORY_FEATURES ?= "image package sdk"
11BUILDHISTORY_DIR ?= "${TOPDIR}/buildhistory"
12BUILDHISTORY_DIR_IMAGE = "${BUILDHISTORY_DIR}/images/${MACHINE_ARCH}/${TCLIBC}/${IMAGE_BASENAME}"
13BUILDHISTORY_DIR_PACKAGE = "${BUILDHISTORY_DIR}/packages/${MULTIMACH_TARGET_SYS}/${PN}"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050014
15# Setting this to non-empty will remove the old content of the buildhistory as part of
16# the current bitbake invocation and replace it with information about what was built
17# during the build.
18#
19# This is meant to be used in continuous integration (CI) systems when invoking bitbake
20# for full world builds. The effect in that case is that information about packages
21# that no longer get build also gets removed from the buildhistory, which is not
22# the case otherwise.
23#
24# The advantage over manually cleaning the buildhistory outside of bitbake is that
25# the "version-going-backwards" check still works. When relying on that, be careful
26# about failed world builds: they will lead to incomplete information in the
27# buildhistory because information about packages that could not be built will
28# also get removed. A CI system should handle that by discarding the buildhistory
29# of failed builds.
30#
31# The expected usage is via auto.conf, but passing via the command line also works
32# with: BB_ENV_EXTRAWHITE=BUILDHISTORY_RESET BUILDHISTORY_RESET=1
33BUILDHISTORY_RESET ?= ""
34
35BUILDHISTORY_OLD_DIR = "${BUILDHISTORY_DIR}/${@ "old" if "${BUILDHISTORY_RESET}" else ""}"
36BUILDHISTORY_OLD_DIR_PACKAGE = "${BUILDHISTORY_OLD_DIR}/packages/${MULTIMACH_TARGET_SYS}/${PN}"
37BUILDHISTORY_DIR_SDK = "${BUILDHISTORY_DIR}/sdk/${SDK_NAME}${SDK_EXT}/${IMAGE_BASENAME}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050038BUILDHISTORY_IMAGE_FILES ?= "/etc/passwd /etc/group"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050039BUILDHISTORY_SDK_FILES ?= "conf/local.conf conf/bblayers.conf conf/auto.conf conf/locked-sigs.inc conf/devtool.conf"
Brad Bishop316dfdd2018-06-25 12:45:53 -040040BUILDHISTORY_COMMIT ?= "1"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050041BUILDHISTORY_COMMIT_AUTHOR ?= "buildhistory <buildhistory@${DISTRO}>"
42BUILDHISTORY_PUSH_REPO ?= ""
43
44SSTATEPOSTINSTFUNCS_append = " buildhistory_emit_pkghistory"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050045# We want to avoid influencing the signatures of sstate tasks - first the function itself:
Patrick Williamsc124f4f2015-09-15 14:41:29 -050046sstate_install[vardepsexclude] += "buildhistory_emit_pkghistory"
47# then the value added to SSTATEPOSTINSTFUNCS:
48SSTATEPOSTINSTFUNCS[vardepvalueexclude] .= "| buildhistory_emit_pkghistory"
49
Brad Bishop6e60e8b2018-02-01 10:27:11 -050050# Similarly for our function that gets the output signatures
51SSTATEPOSTUNPACKFUNCS_append = " buildhistory_emit_outputsigs"
52sstate_installpkgdir[vardepsexclude] += "buildhistory_emit_outputsigs"
53SSTATEPOSTUNPACKFUNCS[vardepvalueexclude] .= "| buildhistory_emit_outputsigs"
54
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050055# All items excepts those listed here will be removed from a recipe's
56# build history directory by buildhistory_emit_pkghistory(). This is
57# necessary because some of these items (package directories, files that
58# we no longer emit) might be obsolete.
Patrick Williamsc124f4f2015-09-15 14:41:29 -050059#
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050060# When extending build history, derive your class from buildhistory.bbclass
61# and extend this list here with the additional files created by the derived
62# class.
Brad Bishop96ff1982019-08-19 13:50:42 -040063BUILDHISTORY_PRESERVE = "latest latest_srcrev sysroot"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050064
Patrick Williamsc0f7c042017-02-23 20:41:17 -060065PATCH_GIT_USER_EMAIL ?= "buildhistory@oe"
66PATCH_GIT_USER_NAME ?= "OpenEmbedded"
67
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050068#
Brad Bishop96ff1982019-08-19 13:50:42 -040069# Write out the contents of the sysroot
70#
71buildhistory_emit_sysroot() {
72 mkdir --parents ${BUILDHISTORY_DIR_PACKAGE}
73 case ${CLASSOVERRIDE} in
74 class-native|class-cross|class-crosssdk)
75 BASE=${SYSROOT_DESTDIR}/${STAGING_DIR_NATIVE}
76 ;;
77 *)
78 BASE=${SYSROOT_DESTDIR}
79 ;;
80 esac
81 buildhistory_list_files_no_owners $BASE ${BUILDHISTORY_DIR_PACKAGE}/sysroot
82}
83
84#
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050085# Write out metadata about this package for comparison when writing future packages
Patrick Williamsc124f4f2015-09-15 14:41:29 -050086#
87python buildhistory_emit_pkghistory() {
Brad Bishop96ff1982019-08-19 13:50:42 -040088 if d.getVar('BB_CURRENTTASK') in ['populate_sysroot', 'populate_sysroot_setscene']:
89 bb.build.exec_func("buildhistory_emit_sysroot", d)
90
Brad Bishop6e60e8b2018-02-01 10:27:11 -050091 if not d.getVar('BB_CURRENTTASK') in ['packagedata', 'packagedata_setscene']:
Patrick Williamsc124f4f2015-09-15 14:41:29 -050092 return 0
93
Brad Bishop6e60e8b2018-02-01 10:27:11 -050094 if not "package" in (d.getVar('BUILDHISTORY_FEATURES') or "").split():
Patrick Williamsc124f4f2015-09-15 14:41:29 -050095 return 0
96
97 import re
98 import json
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080099 import shlex
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500100 import errno
101
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500102 pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE')
103 oldpkghistdir = d.getVar('BUILDHISTORY_OLD_DIR_PACKAGE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500104
105 class RecipeInfo:
106 def __init__(self, name):
107 self.name = name
108 self.pe = "0"
109 self.pv = "0"
110 self.pr = "r0"
111 self.depends = ""
112 self.packages = ""
113 self.srcrev = ""
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500114 self.layer = ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500115
116
117 class PackageInfo:
118 def __init__(self, name):
119 self.name = name
120 self.pe = "0"
121 self.pv = "0"
122 self.pr = "r0"
123 # pkg/pkge/pkgv/pkgr should be empty because we want to be able to default them
124 self.pkg = ""
125 self.pkge = ""
126 self.pkgv = ""
127 self.pkgr = ""
128 self.size = 0
129 self.depends = ""
130 self.rprovides = ""
131 self.rdepends = ""
132 self.rrecommends = ""
133 self.rsuggests = ""
134 self.rreplaces = ""
135 self.rconflicts = ""
136 self.files = ""
137 self.filelist = ""
138 # Variables that need to be written to their own separate file
139 self.filevars = dict.fromkeys(['pkg_preinst', 'pkg_postinst', 'pkg_prerm', 'pkg_postrm'])
140
141 # Should check PACKAGES here to see if anything removed
142
143 def readPackageInfo(pkg, histfile):
144 pkginfo = PackageInfo(pkg)
145 with open(histfile, "r") as f:
146 for line in f:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500147 lns = line.split('=', 1)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500148 name = lns[0].strip()
149 value = lns[1].strip(" \t\r\n").strip('"')
150 if name == "PE":
151 pkginfo.pe = value
152 elif name == "PV":
153 pkginfo.pv = value
154 elif name == "PR":
155 pkginfo.pr = value
156 elif name == "PKG":
157 pkginfo.pkg = value
158 elif name == "PKGE":
159 pkginfo.pkge = value
160 elif name == "PKGV":
161 pkginfo.pkgv = value
162 elif name == "PKGR":
163 pkginfo.pkgr = value
164 elif name == "RPROVIDES":
165 pkginfo.rprovides = value
166 elif name == "RDEPENDS":
167 pkginfo.rdepends = value
168 elif name == "RRECOMMENDS":
169 pkginfo.rrecommends = value
170 elif name == "RSUGGESTS":
171 pkginfo.rsuggests = value
172 elif name == "RREPLACES":
173 pkginfo.rreplaces = value
174 elif name == "RCONFLICTS":
175 pkginfo.rconflicts = value
176 elif name == "PKGSIZE":
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600177 pkginfo.size = int(value)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500178 elif name == "FILES":
179 pkginfo.files = value
180 elif name == "FILELIST":
181 pkginfo.filelist = value
182 # Apply defaults
183 if not pkginfo.pkg:
184 pkginfo.pkg = pkginfo.name
185 if not pkginfo.pkge:
186 pkginfo.pkge = pkginfo.pe
187 if not pkginfo.pkgv:
188 pkginfo.pkgv = pkginfo.pv
189 if not pkginfo.pkgr:
190 pkginfo.pkgr = pkginfo.pr
191 return pkginfo
192
193 def getlastpkgversion(pkg):
194 try:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500195 histfile = os.path.join(oldpkghistdir, pkg, "latest")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500196 return readPackageInfo(pkg, histfile)
197 except EnvironmentError:
198 return None
199
200 def sortpkglist(string):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500201 pkgiter = re.finditer(r'[a-zA-Z0-9.+-]+( \([><=]+[^)]+\))?', string, 0)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500202 pkglist = [p.group(0) for p in pkgiter]
203 pkglist.sort()
204 return ' '.join(pkglist)
205
206 def sortlist(string):
207 items = string.split(' ')
208 items.sort()
209 return ' '.join(items)
210
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500211 pn = d.getVar('PN')
212 pe = d.getVar('PE') or "0"
213 pv = d.getVar('PV')
214 pr = d.getVar('PR')
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500215 layer = bb.utils.get_file_layer(d.getVar('FILE'), d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500216
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500217 pkgdata_dir = d.getVar('PKGDATA_DIR')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500218 packages = ""
219 try:
220 with open(os.path.join(pkgdata_dir, pn)) as f:
221 for line in f.readlines():
222 if line.startswith('PACKAGES: '):
223 packages = oe.utils.squashspaces(line.split(': ', 1)[1])
224 break
225 except IOError as e:
226 if e.errno == errno.ENOENT:
227 # Probably a -cross recipe, just ignore
228 return 0
229 else:
230 raise
231
232 packagelist = packages.split()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500233 preserve = d.getVar('BUILDHISTORY_PRESERVE').split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500234 if not os.path.exists(pkghistdir):
235 bb.utils.mkdirhier(pkghistdir)
236 else:
237 # Remove files for packages that no longer exist
238 for item in os.listdir(pkghistdir):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500239 if item not in preserve:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500240 if item not in packagelist:
241 itempath = os.path.join(pkghistdir, item)
242 if os.path.isdir(itempath):
243 for subfile in os.listdir(itempath):
244 os.unlink(os.path.join(itempath, subfile))
245 os.rmdir(itempath)
246 else:
247 os.unlink(itempath)
248
249 rcpinfo = RecipeInfo(pn)
250 rcpinfo.pe = pe
251 rcpinfo.pv = pv
252 rcpinfo.pr = pr
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500253 rcpinfo.depends = sortlist(oe.utils.squashspaces(d.getVar('DEPENDS') or ""))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500254 rcpinfo.packages = packages
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500255 rcpinfo.layer = layer
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500256 write_recipehistory(rcpinfo, d)
257
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500258 pkgdest = d.getVar('PKGDEST')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500259 for pkg in packagelist:
260 pkgdata = {}
261 with open(os.path.join(pkgdata_dir, 'runtime', pkg)) as f:
262 for line in f.readlines():
263 item = line.rstrip('\n').split(': ', 1)
264 key = item[0]
265 if key.endswith('_' + pkg):
266 key = key[:-len(pkg)-1]
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800267 pkgdata[key] = item[1].encode('latin-1').decode('unicode_escape')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500268
269 pkge = pkgdata.get('PKGE', '0')
270 pkgv = pkgdata['PKGV']
271 pkgr = pkgdata['PKGR']
272 #
273 # Find out what the last version was
274 # Make sure the version did not decrease
275 #
276 lastversion = getlastpkgversion(pkg)
277 if lastversion:
278 last_pkge = lastversion.pkge
279 last_pkgv = lastversion.pkgv
280 last_pkgr = lastversion.pkgr
281 r = bb.utils.vercmp((pkge, pkgv, pkgr), (last_pkge, last_pkgv, last_pkgr))
282 if r < 0:
283 msg = "Package version for package %s went backwards which would break package feeds from (%s:%s-%s to %s:%s-%s)" % (pkg, last_pkge, last_pkgv, last_pkgr, pkge, pkgv, pkgr)
284 package_qa_handle_error("version-going-backwards", msg, d)
285
286 pkginfo = PackageInfo(pkg)
287 # Apparently the version can be different on a per-package basis (see Python)
288 pkginfo.pe = pkgdata.get('PE', '0')
289 pkginfo.pv = pkgdata['PV']
290 pkginfo.pr = pkgdata['PR']
291 pkginfo.pkg = pkgdata['PKG']
292 pkginfo.pkge = pkge
293 pkginfo.pkgv = pkgv
294 pkginfo.pkgr = pkgr
295 pkginfo.rprovides = sortpkglist(oe.utils.squashspaces(pkgdata.get('RPROVIDES', "")))
296 pkginfo.rdepends = sortpkglist(oe.utils.squashspaces(pkgdata.get('RDEPENDS', "")))
297 pkginfo.rrecommends = sortpkglist(oe.utils.squashspaces(pkgdata.get('RRECOMMENDS', "")))
298 pkginfo.rsuggests = sortpkglist(oe.utils.squashspaces(pkgdata.get('RSUGGESTS', "")))
299 pkginfo.rreplaces = sortpkglist(oe.utils.squashspaces(pkgdata.get('RREPLACES', "")))
300 pkginfo.rconflicts = sortpkglist(oe.utils.squashspaces(pkgdata.get('RCONFLICTS', "")))
301 pkginfo.files = oe.utils.squashspaces(pkgdata.get('FILES', ""))
302 for filevar in pkginfo.filevars:
303 pkginfo.filevars[filevar] = pkgdata.get(filevar, "")
304
305 # Gather information about packaged files
306 val = pkgdata.get('FILES_INFO', '')
307 dictval = json.loads(val)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600308 filelist = list(dictval.keys())
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500309 filelist.sort()
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800310 pkginfo.filelist = " ".join([shlex.quote(x) for x in filelist])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500311
312 pkginfo.size = int(pkgdata['PKGSIZE'])
313
314 write_pkghistory(pkginfo, d)
315
316 # Create files-in-<package-name>.txt files containing a list of files of each recipe's package
317 bb.build.exec_func("buildhistory_list_pkg_files", d)
318}
319
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500320python buildhistory_emit_outputsigs() {
321 if not "task" in (d.getVar('BUILDHISTORY_FEATURES') or "").split():
322 return
323
324 import hashlib
325
326 taskoutdir = os.path.join(d.getVar('BUILDHISTORY_DIR'), 'task', 'output')
327 bb.utils.mkdirhier(taskoutdir)
328 currenttask = d.getVar('BB_CURRENTTASK')
329 pn = d.getVar('PN')
330 taskfile = os.path.join(taskoutdir, '%s.%s' % (pn, currenttask))
331
332 cwd = os.getcwd()
333 filesigs = {}
334 for root, _, files in os.walk(cwd):
335 for fname in files:
336 if fname == 'fixmepath':
337 continue
338 fullpath = os.path.join(root, fname)
339 try:
340 if os.path.islink(fullpath):
341 sha256 = hashlib.sha256(os.readlink(fullpath).encode('utf-8')).hexdigest()
342 elif os.path.isfile(fullpath):
343 sha256 = bb.utils.sha256_file(fullpath)
344 else:
345 continue
346 except OSError:
347 bb.warn('buildhistory: unable to read %s to get output signature' % fullpath)
348 continue
349 filesigs[os.path.relpath(fullpath, cwd)] = sha256
350 with open(taskfile, 'w') as f:
351 for fpath, fsig in sorted(filesigs.items(), key=lambda item: item[0]):
352 f.write('%s %s\n' % (fpath, fsig))
353}
354
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500355
356def write_recipehistory(rcpinfo, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500357 bb.debug(2, "Writing recipe history")
358
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500359 pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500360
361 infofile = os.path.join(pkghistdir, "latest")
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600362 with open(infofile, "w") as f:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500363 if rcpinfo.pe != "0":
364 f.write(u"PE = %s\n" % rcpinfo.pe)
365 f.write(u"PV = %s\n" % rcpinfo.pv)
366 f.write(u"PR = %s\n" % rcpinfo.pr)
367 f.write(u"DEPENDS = %s\n" % rcpinfo.depends)
368 f.write(u"PACKAGES = %s\n" % rcpinfo.packages)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500369 f.write(u"LAYER = %s\n" % rcpinfo.layer)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500370
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500371 write_latest_srcrev(d, pkghistdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500372
373def write_pkghistory(pkginfo, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500374 bb.debug(2, "Writing package history for package %s" % pkginfo.name)
375
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500376 pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500377
378 pkgpath = os.path.join(pkghistdir, pkginfo.name)
379 if not os.path.exists(pkgpath):
380 bb.utils.mkdirhier(pkgpath)
381
382 infofile = os.path.join(pkgpath, "latest")
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600383 with open(infofile, "w") as f:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500384 if pkginfo.pe != "0":
385 f.write(u"PE = %s\n" % pkginfo.pe)
386 f.write(u"PV = %s\n" % pkginfo.pv)
387 f.write(u"PR = %s\n" % pkginfo.pr)
388
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600389 if pkginfo.pkg != pkginfo.name:
390 f.write(u"PKG = %s\n" % pkginfo.pkg)
391 if pkginfo.pkge != pkginfo.pe:
392 f.write(u"PKGE = %s\n" % pkginfo.pkge)
393 if pkginfo.pkgv != pkginfo.pv:
394 f.write(u"PKGV = %s\n" % pkginfo.pkgv)
395 if pkginfo.pkgr != pkginfo.pr:
396 f.write(u"PKGR = %s\n" % pkginfo.pkgr)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500397 f.write(u"RPROVIDES = %s\n" % pkginfo.rprovides)
398 f.write(u"RDEPENDS = %s\n" % pkginfo.rdepends)
399 f.write(u"RRECOMMENDS = %s\n" % pkginfo.rrecommends)
400 if pkginfo.rsuggests:
401 f.write(u"RSUGGESTS = %s\n" % pkginfo.rsuggests)
402 if pkginfo.rreplaces:
403 f.write(u"RREPLACES = %s\n" % pkginfo.rreplaces)
404 if pkginfo.rconflicts:
405 f.write(u"RCONFLICTS = %s\n" % pkginfo.rconflicts)
406 f.write(u"PKGSIZE = %d\n" % pkginfo.size)
407 f.write(u"FILES = %s\n" % pkginfo.files)
408 f.write(u"FILELIST = %s\n" % pkginfo.filelist)
409
410 for filevar in pkginfo.filevars:
411 filevarpath = os.path.join(pkgpath, "latest.%s" % filevar)
412 val = pkginfo.filevars[filevar]
413 if val:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600414 with open(filevarpath, "w") as f:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500415 f.write(val)
416 else:
417 if os.path.exists(filevarpath):
418 os.unlink(filevarpath)
419
420#
421# rootfs_type can be: image, sdk_target, sdk_host
422#
423def buildhistory_list_installed(d, rootfs_type="image"):
424 from oe.rootfs import image_list_installed_packages
425 from oe.sdk import sdk_list_installed_packages
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500426 from oe.utils import format_pkg_list
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500427
428 process_list = [('file', 'bh_installed_pkgs.txt'),\
429 ('deps', 'bh_installed_pkgs_deps.txt')]
430
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500431 if rootfs_type == "image":
432 pkgs = image_list_installed_packages(d)
433 else:
434 pkgs = sdk_list_installed_packages(d, rootfs_type == "sdk_target")
435
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500436 for output_type, output_file in process_list:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500437 output_file_full = os.path.join(d.getVar('WORKDIR'), output_file)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500438
439 with open(output_file_full, 'w') as output:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500440 output.write(format_pkg_list(pkgs, output_type))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500441
442python buildhistory_list_installed_image() {
443 buildhistory_list_installed(d)
444}
445
446python buildhistory_list_installed_sdk_target() {
447 buildhistory_list_installed(d, "sdk_target")
448}
449
450python buildhistory_list_installed_sdk_host() {
451 buildhistory_list_installed(d, "sdk_host")
452}
453
454buildhistory_get_installed() {
455 mkdir -p $1
456
457 # Get list of installed packages
458 pkgcache="$1/installed-packages.tmp"
459 cat ${WORKDIR}/bh_installed_pkgs.txt | sort > $pkgcache && rm ${WORKDIR}/bh_installed_pkgs.txt
460
461 cat $pkgcache | awk '{ print $1 }' > $1/installed-package-names.txt
462 if [ -s $pkgcache ] ; then
463 cat $pkgcache | awk '{ print $2 }' | xargs -n1 basename > $1/installed-packages.txt
464 else
465 printf "" > $1/installed-packages.txt
466 fi
467
468 # Produce dependency graph
469 # First, quote each name to handle characters that cause issues for dot
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500470 sed 's:\([^| ]*\):"\1":g' ${WORKDIR}/bh_installed_pkgs_deps.txt > $1/depends.tmp &&
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500471 rm ${WORKDIR}/bh_installed_pkgs_deps.txt
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500472 # Remove lines with rpmlib(...) and config(...) dependencies, change the
473 # delimiter from pipe to "->", set the style for recommend lines and
474 # turn versioned dependencies into edge labels.
475 sed -i -e '/rpmlib(/d' \
476 -e '/config(/d' \
477 -e 's:|: -> :' \
478 -e 's:"\[REC\]":[style=dotted]:' \
479 -e 's:"\([<>=]\+\)" "\([^"]*\)":[label="\1 \2"]:' \
480 $1/depends.tmp
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500481 # Add header, sorted and de-duped contents and footer and then delete the temp file
482 printf "digraph depends {\n node [shape=plaintext]\n" > $1/depends.dot
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500483 cat $1/depends.tmp | sort -u >> $1/depends.dot
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500484 echo "}" >> $1/depends.dot
485 rm $1/depends.tmp
486
487 # Produce installed package sizes list
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500488 oe-pkgdata-util -p ${PKGDATA_DIR} read-value "PKGSIZE" -n -f $pkgcache > $1/installed-package-sizes.tmp
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500489 cat $1/installed-package-sizes.tmp | awk '{print $2 "\tKiB\t" $1}' | sort -n -r > $1/installed-package-sizes.txt
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500490 rm $1/installed-package-sizes.tmp
491
492 # We're now done with the cache, delete it
493 rm $pkgcache
494
495 if [ "$2" != "sdk" ] ; then
496 # Produce some cut-down graphs (for readability)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500497 grep -v kernel-image $1/depends.dot | grep -v kernel-3 | grep -v kernel-4 > $1/depends-nokernel.dot
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500498 grep -v libc6 $1/depends-nokernel.dot | grep -v libgcc > $1/depends-nokernel-nolibc.dot
499 grep -v update- $1/depends-nokernel-nolibc.dot > $1/depends-nokernel-nolibc-noupdate.dot
500 grep -v kernel-module $1/depends-nokernel-nolibc-noupdate.dot > $1/depends-nokernel-nolibc-noupdate-nomodules.dot
501 fi
502
503 # add complementary package information
504 if [ -e ${WORKDIR}/complementary_pkgs.txt ]; then
505 cp ${WORKDIR}/complementary_pkgs.txt $1
506 fi
507}
508
509buildhistory_get_image_installed() {
510 # Anything requiring the use of the packaging system should be done in here
511 # in case the packaging files are going to be removed for this image
512
513 if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'image', '1', '0', d)}" = "0" ] ; then
514 return
515 fi
516
517 buildhistory_get_installed ${BUILDHISTORY_DIR_IMAGE}
518}
519
520buildhistory_get_sdk_installed() {
521 # Anything requiring the use of the packaging system should be done in here
522 # in case the packaging files are going to be removed for this SDK
523
524 if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'sdk', '1', '0', d)}" = "0" ] ; then
525 return
526 fi
527
528 buildhistory_get_installed ${BUILDHISTORY_DIR_SDK}/$1 sdk
529}
530
531buildhistory_get_sdk_installed_host() {
532 buildhistory_get_sdk_installed host
533}
534
535buildhistory_get_sdk_installed_target() {
536 buildhistory_get_sdk_installed target
537}
538
539buildhistory_list_files() {
540 # List the files in the specified directory, but exclude date/time etc.
Brad Bishop19323692019-04-05 15:28:33 -0400541 # This is somewhat messy, but handles where the size is not printed for device files under pseudo
542 ( cd $1
543 find_cmd='find . ! -path . -printf "%M %-10u %-10g %10s %p -> %l\n"'
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500544 if [ "$3" = "fakeroot" ] ; then
Brad Bishop19323692019-04-05 15:28:33 -0400545 eval ${FAKEROOTENV} ${FAKEROOTCMD} $find_cmd
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500546 else
Brad Bishop19323692019-04-05 15:28:33 -0400547 eval $find_cmd
548 fi | sort -k5 | sed 's/ * -> $//' > $2 )
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500549}
550
Brad Bishop96ff1982019-08-19 13:50:42 -0400551buildhistory_list_files_no_owners() {
552 # List the files in the specified directory, but exclude date/time etc.
553 # Also don't output the ownership data, but instead output just - - so
554 # that the same parsing code as for _list_files works.
555 # This is somewhat messy, but handles where the size is not printed for device files under pseudo
556 ( cd $1
557 find_cmd='find . ! -path . -printf "%M - - %10s %p -> %l\n"'
558 if [ "$3" = "fakeroot" ] ; then
559 eval ${FAKEROOTENV} ${FAKEROOTCMD} "$find_cmd"
560 else
561 eval "$find_cmd"
562 fi | sort -k5 | sed 's/ * -> $//' > $2 )
563}
564
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500565buildhistory_list_pkg_files() {
566 # Create individual files-in-package for each recipe's package
567 for pkgdir in $(find ${PKGDEST}/* -maxdepth 0 -type d); do
568 pkgname=$(basename $pkgdir)
569 outfolder="${BUILDHISTORY_DIR_PACKAGE}/$pkgname"
570 outfile="$outfolder/files-in-package.txt"
571 # Make sure the output folder exists so we can create the file
572 if [ ! -d $outfolder ] ; then
573 bbdebug 2 "Folder $outfolder does not exist, file $outfile not created"
574 continue
575 fi
576 buildhistory_list_files $pkgdir $outfile fakeroot
577 done
578}
579
580buildhistory_get_imageinfo() {
581 if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'image', '1', '0', d)}" = "0" ] ; then
582 return
583 fi
584
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500585 mkdir -p ${BUILDHISTORY_DIR_IMAGE}
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500586 buildhistory_list_files ${IMAGE_ROOTFS} ${BUILDHISTORY_DIR_IMAGE}/files-in-image.txt
587
588 # Collect files requested in BUILDHISTORY_IMAGE_FILES
589 rm -rf ${BUILDHISTORY_DIR_IMAGE}/image-files
590 for f in ${BUILDHISTORY_IMAGE_FILES}; do
591 if [ -f ${IMAGE_ROOTFS}/$f ] ; then
592 mkdir -p ${BUILDHISTORY_DIR_IMAGE}/image-files/`dirname $f`
593 cp ${IMAGE_ROOTFS}/$f ${BUILDHISTORY_DIR_IMAGE}/image-files/$f
594 fi
595 done
596
597 # Record some machine-readable meta-information about the image
598 printf "" > ${BUILDHISTORY_DIR_IMAGE}/image-info.txt
599 cat >> ${BUILDHISTORY_DIR_IMAGE}/image-info.txt <<END
600${@buildhistory_get_imagevars(d)}
601END
602 imagesize=`du -ks ${IMAGE_ROOTFS} | awk '{ print $1 }'`
603 echo "IMAGESIZE = $imagesize" >> ${BUILDHISTORY_DIR_IMAGE}/image-info.txt
604
605 # Add some configuration information
606 echo "${MACHINE}: ${IMAGE_BASENAME} configured for ${DISTRO} ${DISTRO_VERSION}" > ${BUILDHISTORY_DIR_IMAGE}/build-id.txt
607
608 cat >> ${BUILDHISTORY_DIR_IMAGE}/build-id.txt <<END
609${@buildhistory_get_build_id(d)}
610END
611}
612
613buildhistory_get_sdkinfo() {
614 if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'sdk', '1', '0', d)}" = "0" ] ; then
615 return
616 fi
617
618 buildhistory_list_files ${SDK_OUTPUT} ${BUILDHISTORY_DIR_SDK}/files-in-sdk.txt
619
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500620 # Collect files requested in BUILDHISTORY_SDK_FILES
621 rm -rf ${BUILDHISTORY_DIR_SDK}/sdk-files
622 for f in ${BUILDHISTORY_SDK_FILES}; do
623 if [ -f ${SDK_OUTPUT}/${SDKPATH}/$f ] ; then
624 mkdir -p ${BUILDHISTORY_DIR_SDK}/sdk-files/`dirname $f`
625 cp ${SDK_OUTPUT}/${SDKPATH}/$f ${BUILDHISTORY_DIR_SDK}/sdk-files/$f
626 fi
627 done
628
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500629 # Record some machine-readable meta-information about the SDK
630 printf "" > ${BUILDHISTORY_DIR_SDK}/sdk-info.txt
631 cat >> ${BUILDHISTORY_DIR_SDK}/sdk-info.txt <<END
632${@buildhistory_get_sdkvars(d)}
633END
634 sdksize=`du -ks ${SDK_OUTPUT} | awk '{ print $1 }'`
635 echo "SDKSIZE = $sdksize" >> ${BUILDHISTORY_DIR_SDK}/sdk-info.txt
636}
637
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500638python buildhistory_get_extra_sdkinfo() {
639 import operator
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500640 from oe.sdk import get_extra_sdkinfo
641
642 sstate_dir = d.expand('${SDK_OUTPUT}/${SDKPATH}/sstate-cache')
643 extra_info = get_extra_sdkinfo(sstate_dir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500644
645 if d.getVar('BB_CURRENTTASK') == 'populate_sdk_ext' and \
646 "sdk" in (d.getVar('BUILDHISTORY_FEATURES') or "").split():
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500647 with open(d.expand('${BUILDHISTORY_DIR_SDK}/sstate-package-sizes.txt'), 'w') as f:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500648 filesizes_sorted = sorted(extra_info['filesizes'].items(), key=operator.itemgetter(1, 0), reverse=True)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500649 for fn, size in filesizes_sorted:
650 f.write('%10d KiB %s\n' % (size, fn))
651 with open(d.expand('${BUILDHISTORY_DIR_SDK}/sstate-task-sizes.txt'), 'w') as f:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500652 tasksizes_sorted = sorted(extra_info['tasksizes'].items(), key=operator.itemgetter(1, 0), reverse=True)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500653 for task, size in tasksizes_sorted:
654 f.write('%10d KiB %s\n' % (size, task))
655}
656
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500657# By using ROOTFS_POSTUNINSTALL_COMMAND we get in after uninstallation of
658# unneeded packages but before the removal of packaging files
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500659ROOTFS_POSTUNINSTALL_COMMAND += "buildhistory_list_installed_image ;"
660ROOTFS_POSTUNINSTALL_COMMAND += "buildhistory_get_image_installed ;"
661ROOTFS_POSTUNINSTALL_COMMAND[vardepvalueexclude] .= "| buildhistory_list_installed_image ;| buildhistory_get_image_installed ;"
662ROOTFS_POSTUNINSTALL_COMMAND[vardepsexclude] += "buildhistory_list_installed_image buildhistory_get_image_installed"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500663
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500664IMAGE_POSTPROCESS_COMMAND += "buildhistory_get_imageinfo ;"
665IMAGE_POSTPROCESS_COMMAND[vardepvalueexclude] .= "| buildhistory_get_imageinfo ;"
666IMAGE_POSTPROCESS_COMMAND[vardepsexclude] += "buildhistory_get_imageinfo"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500667
668# We want these to be the last run so that we get called after complementary package installation
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500669POPULATE_SDK_POST_TARGET_COMMAND_append = " buildhistory_list_installed_sdk_target;"
670POPULATE_SDK_POST_TARGET_COMMAND_append = " buildhistory_get_sdk_installed_target;"
671POPULATE_SDK_POST_TARGET_COMMAND[vardepvalueexclude] .= "| buildhistory_list_installed_sdk_target;| buildhistory_get_sdk_installed_target;"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500672
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500673POPULATE_SDK_POST_HOST_COMMAND_append = " buildhistory_list_installed_sdk_host;"
674POPULATE_SDK_POST_HOST_COMMAND_append = " buildhistory_get_sdk_installed_host;"
675POPULATE_SDK_POST_HOST_COMMAND[vardepvalueexclude] .= "| buildhistory_list_installed_sdk_host;| buildhistory_get_sdk_installed_host;"
676
677SDK_POSTPROCESS_COMMAND_append = " buildhistory_get_sdkinfo ; buildhistory_get_extra_sdkinfo; "
678SDK_POSTPROCESS_COMMAND[vardepvalueexclude] .= "| buildhistory_get_sdkinfo ; buildhistory_get_extra_sdkinfo; "
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500679
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500680python buildhistory_write_sigs() {
681 if not "task" in (d.getVar('BUILDHISTORY_FEATURES') or "").split():
682 return
683
684 # Create sigs file
685 if hasattr(bb.parse.siggen, 'dump_siglist'):
686 taskoutdir = os.path.join(d.getVar('BUILDHISTORY_DIR'), 'task')
687 bb.utils.mkdirhier(taskoutdir)
688 bb.parse.siggen.dump_siglist(os.path.join(taskoutdir, 'tasksigs.txt'))
689}
690
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500691def buildhistory_get_build_id(d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500692 if d.getVar('BB_WORKERCONTEXT') != '1':
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500693 return ""
694 localdata = bb.data.createCopy(d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500695 statuslines = []
696 for func in oe.data.typed_value('BUILDCFG_FUNCS', localdata):
697 g = globals()
698 if func not in g:
699 bb.warn("Build configuration function '%s' does not exist" % func)
700 else:
701 flines = g[func](localdata)
702 if flines:
703 statuslines.extend(flines)
704
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500705 statusheader = d.getVar('BUILDCFG_HEADER')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500706 return('\n%s\n%s\n' % (statusheader, '\n'.join(statuslines)))
707
Brad Bishop19323692019-04-05 15:28:33 -0400708def buildhistory_get_modified(path):
709 # copied from get_layer_git_status() in image-buildinfo.bbclass
710 import subprocess
711 try:
712 subprocess.check_output("""cd %s; export PSEUDO_UNLOAD=1; set -e;
713 git diff --quiet --no-ext-diff
714 git diff --quiet --no-ext-diff --cached""" % path,
715 shell=True,
716 stderr=subprocess.STDOUT)
717 return ""
718 except subprocess.CalledProcessError as ex:
719 # Silently treat errors as "modified", without checking for the
720 # (expected) return code 1 in a modified git repo. For example, we get
721 # output and a 129 return code when a layer isn't a git repo at all.
722 return " -- modified"
723
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500724def buildhistory_get_metadata_revs(d):
725 # We want an easily machine-readable format here, so get_layers_branch_rev isn't quite what we want
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500726 layers = (d.getVar("BBLAYERS") or "").split()
Brad Bishop19323692019-04-05 15:28:33 -0400727 medadata_revs = ["%-17s = %s:%s%s" % (os.path.basename(i), \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500728 base_get_metadata_git_branch(i, None).strip(), \
Brad Bishop19323692019-04-05 15:28:33 -0400729 base_get_metadata_git_revision(i, None), \
730 buildhistory_get_modified(i)) \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500731 for i in layers]
732 return '\n'.join(medadata_revs)
733
734def outputvars(vars, listvars, d):
735 vars = vars.split()
736 listvars = listvars.split()
737 ret = ""
738 for var in vars:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500739 value = d.getVar(var) or ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500740 if var in listvars:
741 # Squash out spaces
742 value = oe.utils.squashspaces(value)
743 ret += "%s = %s\n" % (var, value)
744 return ret.rstrip('\n')
745
746def buildhistory_get_imagevars(d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500747 if d.getVar('BB_WORKERCONTEXT') != '1':
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500748 return ""
749 imagevars = "DISTRO DISTRO_VERSION USER_CLASSES IMAGE_CLASSES IMAGE_FEATURES IMAGE_LINGUAS IMAGE_INSTALL BAD_RECOMMENDATIONS NO_RECOMMENDATIONS PACKAGE_EXCLUDE ROOTFS_POSTPROCESS_COMMAND IMAGE_POSTPROCESS_COMMAND"
750 listvars = "USER_CLASSES IMAGE_CLASSES IMAGE_FEATURES IMAGE_LINGUAS IMAGE_INSTALL BAD_RECOMMENDATIONS PACKAGE_EXCLUDE"
751 return outputvars(imagevars, listvars, d)
752
753def buildhistory_get_sdkvars(d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500754 if d.getVar('BB_WORKERCONTEXT') != '1':
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500755 return ""
756 sdkvars = "DISTRO DISTRO_VERSION SDK_NAME SDK_VERSION SDKMACHINE SDKIMAGE_FEATURES BAD_RECOMMENDATIONS NO_RECOMMENDATIONS PACKAGE_EXCLUDE"
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500757 if d.getVar('BB_CURRENTTASK') == 'populate_sdk_ext':
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500758 # Extensible SDK uses some additional variables
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600759 sdkvars += " SDK_LOCAL_CONF_WHITELIST SDK_LOCAL_CONF_BLACKLIST SDK_INHERIT_BLACKLIST SDK_UPDATE_URL SDK_EXT_TYPE SDK_RECRDEP_TASKS SDK_INCLUDE_PKGDATA SDK_INCLUDE_TOOLCHAIN"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500760 listvars = "SDKIMAGE_FEATURES BAD_RECOMMENDATIONS PACKAGE_EXCLUDE SDK_LOCAL_CONF_WHITELIST SDK_LOCAL_CONF_BLACKLIST SDK_INHERIT_BLACKLIST"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500761 return outputvars(sdkvars, listvars, d)
762
763
764def buildhistory_get_cmdline(d):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500765 argv = d.getVar('BB_CMDLINE', False)
766 if argv:
767 if argv[0].endswith('bin/bitbake'):
768 bincmd = 'bitbake'
769 else:
770 bincmd = argv[0]
771 return '%s %s' % (bincmd, ' '.join(argv[1:]))
772 return ''
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500773
774
775buildhistory_single_commit() {
776 if [ "$3" = "" ] ; then
777 commitopts="${BUILDHISTORY_DIR}/ --allow-empty"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500778 shortlogprefix="No changes: "
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500779 else
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500780 commitopts=""
781 shortlogprefix=""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500782 fi
783 if [ "${BUILDHISTORY_BUILD_FAILURES}" = "0" ] ; then
784 result="succeeded"
785 else
786 result="failed"
787 fi
788 case ${BUILDHISTORY_BUILD_INTERRUPTED} in
789 1)
790 result="$result (interrupted)"
791 ;;
792 2)
793 result="$result (force interrupted)"
794 ;;
795 esac
796 commitmsgfile=`mktemp`
797 cat > $commitmsgfile << END
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500798${shortlogprefix}Build ${BUILDNAME} of ${DISTRO} ${DISTRO_VERSION} for machine ${MACHINE} on $2
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500799
800cmd: $1
801
802result: $result
803
804metadata revisions:
805END
806 cat ${BUILDHISTORY_DIR}/metadata-revs >> $commitmsgfile
807 git commit $commitopts -F $commitmsgfile --author "${BUILDHISTORY_COMMIT_AUTHOR}" > /dev/null
808 rm $commitmsgfile
809}
810
811buildhistory_commit() {
812 if [ ! -d ${BUILDHISTORY_DIR} ] ; then
813 # Code above that creates this dir never executed, so there can't be anything to commit
814 return
815 fi
816
817 # Create a machine-readable list of metadata revisions for each layer
818 cat > ${BUILDHISTORY_DIR}/metadata-revs <<END
819${@buildhistory_get_metadata_revs(d)}
820END
821
822 ( cd ${BUILDHISTORY_DIR}/
823 # Initialise the repo if necessary
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500824 if [ ! -e .git ] ; then
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500825 git init -q
826 else
827 git tag -f build-minus-3 build-minus-2 > /dev/null 2>&1 || true
828 git tag -f build-minus-2 build-minus-1 > /dev/null 2>&1 || true
829 git tag -f build-minus-1 > /dev/null 2>&1 || true
830 fi
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600831
832 check_git_config
833
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500834 # Check if there are new/changed files to commit (other than metadata-revs)
835 repostatus=`git status --porcelain | grep -v " metadata-revs$"`
836 HOSTNAME=`hostname 2>/dev/null || echo unknown`
837 CMDLINE="${@buildhistory_get_cmdline(d)}"
838 if [ "$repostatus" != "" ] ; then
839 git add -A .
840 # porcelain output looks like "?? packages/foo/bar"
841 # Ensure we commit metadata-revs with the first commit
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500842 buildhistory_single_commit "$CMDLINE" "$HOSTNAME" dummy
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500843 git gc --auto --quiet
844 else
845 buildhistory_single_commit "$CMDLINE" "$HOSTNAME"
846 fi
847 if [ "${BUILDHISTORY_PUSH_REPO}" != "" ] ; then
848 git push -q ${BUILDHISTORY_PUSH_REPO}
849 fi) || true
850}
851
852python buildhistory_eventhandler() {
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500853 if e.data.getVar('BUILDHISTORY_FEATURES').strip():
854 reset = e.data.getVar("BUILDHISTORY_RESET")
855 olddir = e.data.getVar("BUILDHISTORY_OLD_DIR")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500856 if isinstance(e, bb.event.BuildStarted):
857 if reset:
858 import shutil
859 # Clean up after potentially interrupted build.
860 if os.path.isdir(olddir):
861 shutil.rmtree(olddir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500862 rootdir = e.data.getVar("BUILDHISTORY_DIR")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500863 entries = [ x for x in os.listdir(rootdir) if not x.startswith('.') ]
864 bb.utils.mkdirhier(olddir)
865 for entry in entries:
866 os.rename(os.path.join(rootdir, entry),
867 os.path.join(olddir, entry))
868 elif isinstance(e, bb.event.BuildCompleted):
869 if reset:
870 import shutil
871 shutil.rmtree(olddir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500872 if e.data.getVar("BUILDHISTORY_COMMIT") == "1":
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500873 bb.note("Writing buildhistory")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500874 bb.build.exec_func("buildhistory_write_sigs", d)
Brad Bishopf3fd2882019-06-21 08:06:37 -0400875 import time
876 start=time.time()
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500877 localdata = bb.data.createCopy(e.data)
878 localdata.setVar('BUILDHISTORY_BUILD_FAILURES', str(e._failures))
879 interrupted = getattr(e, '_interrupted', 0)
880 localdata.setVar('BUILDHISTORY_BUILD_INTERRUPTED', str(interrupted))
881 bb.build.exec_func("buildhistory_commit", localdata)
Brad Bishopf3fd2882019-06-21 08:06:37 -0400882 stop=time.time()
883 bb.note("Writing buildhistory took: %s seconds" % round(stop-start))
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500884 else:
885 bb.note("No commit since BUILDHISTORY_COMMIT != '1'")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500886}
887
888addhandler buildhistory_eventhandler
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500889buildhistory_eventhandler[eventmask] = "bb.event.BuildCompleted bb.event.BuildStarted"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500890
891
892# FIXME this ought to be moved into the fetcher
893def _get_srcrev_values(d):
894 """
895 Return the version strings for the current recipe
896 """
897
898 scms = []
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500899 fetcher = bb.fetch.Fetch(d.getVar('SRC_URI').split(), d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500900 urldata = fetcher.ud
901 for u in urldata:
902 if urldata[u].method.supports_srcrev():
903 scms.append(u)
904
905 autoinc_templ = 'AUTOINC+'
906 dict_srcrevs = {}
907 dict_tag_srcrevs = {}
908 for scm in scms:
909 ud = urldata[scm]
910 for name in ud.names:
911 try:
912 rev = ud.method.sortable_revision(ud, d, name)
913 except TypeError:
914 # support old bitbake versions
915 rev = ud.method.sortable_revision(scm, ud, d, name)
916 # Clean this up when we next bump bitbake version
917 if type(rev) != str:
918 autoinc, rev = rev
919 elif rev.startswith(autoinc_templ):
920 rev = rev[len(autoinc_templ):]
921 dict_srcrevs[name] = rev
922 if 'tag' in ud.parm:
923 tag = ud.parm['tag'];
924 key = name+'_'+tag
925 dict_tag_srcrevs[key] = rev
926 return (dict_srcrevs, dict_tag_srcrevs)
927
928do_fetch[postfuncs] += "write_srcrev"
929do_fetch[vardepsexclude] += "write_srcrev"
930python write_srcrev() {
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500931 write_latest_srcrev(d, d.getVar('BUILDHISTORY_DIR_PACKAGE'))
932}
933
934def write_latest_srcrev(d, pkghistdir):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500935 srcrevfile = os.path.join(pkghistdir, 'latest_srcrev')
936
937 srcrevs, tag_srcrevs = _get_srcrev_values(d)
938 if srcrevs:
939 if not os.path.exists(pkghistdir):
940 bb.utils.mkdirhier(pkghistdir)
941 old_tag_srcrevs = {}
942 if os.path.exists(srcrevfile):
943 with open(srcrevfile) as f:
944 for line in f:
945 if line.startswith('# tag_'):
946 key, value = line.split("=", 1)
947 key = key.replace('# tag_', '').strip()
948 value = value.replace('"', '').strip()
949 old_tag_srcrevs[key] = value
950 with open(srcrevfile, 'w') as f:
951 orig_srcrev = d.getVar('SRCREV', False) or 'INVALID'
952 if orig_srcrev != 'INVALID':
953 f.write('# SRCREV = "%s"\n' % orig_srcrev)
954 if len(srcrevs) > 1:
Brad Bishop19323692019-04-05 15:28:33 -0400955 for name, srcrev in sorted(srcrevs.items()):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500956 orig_srcrev = d.getVar('SRCREV_%s' % name, False)
957 if orig_srcrev:
958 f.write('# SRCREV_%s = "%s"\n' % (name, orig_srcrev))
959 f.write('SRCREV_%s = "%s"\n' % (name, srcrev))
960 else:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500961 f.write('SRCREV = "%s"\n' % next(iter(srcrevs.values())))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500962 if len(tag_srcrevs) > 0:
Brad Bishop19323692019-04-05 15:28:33 -0400963 for name, srcrev in sorted(tag_srcrevs.items()):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500964 f.write('# tag_%s = "%s"\n' % (name, srcrev))
965 if name in old_tag_srcrevs and old_tag_srcrevs[name] != srcrev:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500966 pkg = d.getVar('PN')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500967 bb.warn("Revision for tag %s in package %s was changed since last build (from %s to %s)" % (name, pkg, old_tag_srcrevs[name], srcrev))
968
969 else:
970 if os.path.exists(srcrevfile):
971 os.remove(srcrevfile)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500972
973do_testimage[postfuncs] += "write_ptest_result"
974do_testimage[vardepsexclude] += "write_ptest_result"
975
976python write_ptest_result() {
977 write_latest_ptest_result(d, d.getVar('BUILDHISTORY_DIR'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500978}
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500979
980def write_latest_ptest_result(d, histdir):
981 import glob
982 import subprocess
983 test_log_dir = d.getVar('TEST_LOG_DIR')
984 input_ptest = os.path.join(test_log_dir, 'ptest_log')
985 output_ptest = os.path.join(histdir, 'ptest')
986 if os.path.exists(input_ptest):
987 try:
988 # Lock it avoid race issue
989 lock = bb.utils.lockfile(output_ptest + "/ptest.lock")
990 bb.utils.mkdirhier(output_ptest)
991 oe.path.copytree(input_ptest, output_ptest)
992 # Sort test result
993 for result in glob.glob('%s/pass.fail.*' % output_ptest):
994 bb.debug(1, 'Processing %s' % result)
995 cmd = ['sort', result, '-o', result]
996 bb.debug(1, 'Running %s' % cmd)
997 ret = subprocess.call(cmd)
998 if ret != 0:
999 bb.error('Failed to run %s!' % cmd)
1000 finally:
1001 bb.utils.unlockfile(lock)