blob: 7a5534edd8af311d8e3882e27a0f32e04aca5d54 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001#
2# Records history of build output in order to detect regressions
3#
4# Based in part on testlab.bbclass and packagehistory.bbclass
5#
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05006# Copyright (C) 2011-2016 Intel Corporation
Patrick Williamsc124f4f2015-09-15 14:41:29 -05007# Copyright (C) 2007-2011 Koen Kooi <koen@openembedded.org>
8#
9
10BUILDHISTORY_FEATURES ?= "image package sdk"
11BUILDHISTORY_DIR ?= "${TOPDIR}/buildhistory"
12BUILDHISTORY_DIR_IMAGE = "${BUILDHISTORY_DIR}/images/${MACHINE_ARCH}/${TCLIBC}/${IMAGE_BASENAME}"
13BUILDHISTORY_DIR_PACKAGE = "${BUILDHISTORY_DIR}/packages/${MULTIMACH_TARGET_SYS}/${PN}"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050014
15# Setting this to non-empty will remove the old content of the buildhistory as part of
16# the current bitbake invocation and replace it with information about what was built
17# during the build.
18#
19# This is meant to be used in continuous integration (CI) systems when invoking bitbake
20# for full world builds. The effect in that case is that information about packages
21# that no longer get build also gets removed from the buildhistory, which is not
22# the case otherwise.
23#
24# The advantage over manually cleaning the buildhistory outside of bitbake is that
25# the "version-going-backwards" check still works. When relying on that, be careful
26# about failed world builds: they will lead to incomplete information in the
27# buildhistory because information about packages that could not be built will
28# also get removed. A CI system should handle that by discarding the buildhistory
29# of failed builds.
30#
31# The expected usage is via auto.conf, but passing via the command line also works
32# with: BB_ENV_EXTRAWHITE=BUILDHISTORY_RESET BUILDHISTORY_RESET=1
33BUILDHISTORY_RESET ?= ""
34
35BUILDHISTORY_OLD_DIR = "${BUILDHISTORY_DIR}/${@ "old" if "${BUILDHISTORY_RESET}" else ""}"
36BUILDHISTORY_OLD_DIR_PACKAGE = "${BUILDHISTORY_OLD_DIR}/packages/${MULTIMACH_TARGET_SYS}/${PN}"
37BUILDHISTORY_DIR_SDK = "${BUILDHISTORY_DIR}/sdk/${SDK_NAME}${SDK_EXT}/${IMAGE_BASENAME}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050038BUILDHISTORY_IMAGE_FILES ?= "/etc/passwd /etc/group"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050039BUILDHISTORY_SDK_FILES ?= "conf/local.conf conf/bblayers.conf conf/auto.conf conf/locked-sigs.inc conf/devtool.conf"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050040BUILDHISTORY_COMMIT ?= "0"
41BUILDHISTORY_COMMIT_AUTHOR ?= "buildhistory <buildhistory@${DISTRO}>"
42BUILDHISTORY_PUSH_REPO ?= ""
43
44SSTATEPOSTINSTFUNCS_append = " buildhistory_emit_pkghistory"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050045# We want to avoid influencing the signatures of sstate tasks - first the function itself:
Patrick Williamsc124f4f2015-09-15 14:41:29 -050046sstate_install[vardepsexclude] += "buildhistory_emit_pkghistory"
47# then the value added to SSTATEPOSTINSTFUNCS:
48SSTATEPOSTINSTFUNCS[vardepvalueexclude] .= "| buildhistory_emit_pkghistory"
49
Brad Bishop6e60e8b2018-02-01 10:27:11 -050050# Similarly for our function that gets the output signatures
51SSTATEPOSTUNPACKFUNCS_append = " buildhistory_emit_outputsigs"
52sstate_installpkgdir[vardepsexclude] += "buildhistory_emit_outputsigs"
53SSTATEPOSTUNPACKFUNCS[vardepvalueexclude] .= "| buildhistory_emit_outputsigs"
54
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050055# All items excepts those listed here will be removed from a recipe's
56# build history directory by buildhistory_emit_pkghistory(). This is
57# necessary because some of these items (package directories, files that
58# we no longer emit) might be obsolete.
Patrick Williamsc124f4f2015-09-15 14:41:29 -050059#
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050060# When extending build history, derive your class from buildhistory.bbclass
61# and extend this list here with the additional files created by the derived
62# class.
63BUILDHISTORY_PRESERVE = "latest latest_srcrev"
64
Patrick Williamsc0f7c042017-02-23 20:41:17 -060065PATCH_GIT_USER_EMAIL ?= "buildhistory@oe"
66PATCH_GIT_USER_NAME ?= "OpenEmbedded"
67
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050068#
69# Write out metadata about this package for comparison when writing future packages
Patrick Williamsc124f4f2015-09-15 14:41:29 -050070#
71python buildhistory_emit_pkghistory() {
Brad Bishop6e60e8b2018-02-01 10:27:11 -050072 if not d.getVar('BB_CURRENTTASK') in ['packagedata', 'packagedata_setscene']:
Patrick Williamsc124f4f2015-09-15 14:41:29 -050073 return 0
74
Brad Bishop6e60e8b2018-02-01 10:27:11 -050075 if not "package" in (d.getVar('BUILDHISTORY_FEATURES') or "").split():
Patrick Williamsc124f4f2015-09-15 14:41:29 -050076 return 0
77
78 import re
79 import json
80 import errno
81
Brad Bishop6e60e8b2018-02-01 10:27:11 -050082 pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE')
83 oldpkghistdir = d.getVar('BUILDHISTORY_OLD_DIR_PACKAGE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -050084
85 class RecipeInfo:
86 def __init__(self, name):
87 self.name = name
88 self.pe = "0"
89 self.pv = "0"
90 self.pr = "r0"
91 self.depends = ""
92 self.packages = ""
93 self.srcrev = ""
Brad Bishop6e60e8b2018-02-01 10:27:11 -050094 self.layer = ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -050095
96
97 class PackageInfo:
98 def __init__(self, name):
99 self.name = name
100 self.pe = "0"
101 self.pv = "0"
102 self.pr = "r0"
103 # pkg/pkge/pkgv/pkgr should be empty because we want to be able to default them
104 self.pkg = ""
105 self.pkge = ""
106 self.pkgv = ""
107 self.pkgr = ""
108 self.size = 0
109 self.depends = ""
110 self.rprovides = ""
111 self.rdepends = ""
112 self.rrecommends = ""
113 self.rsuggests = ""
114 self.rreplaces = ""
115 self.rconflicts = ""
116 self.files = ""
117 self.filelist = ""
118 # Variables that need to be written to their own separate file
119 self.filevars = dict.fromkeys(['pkg_preinst', 'pkg_postinst', 'pkg_prerm', 'pkg_postrm'])
120
121 # Should check PACKAGES here to see if anything removed
122
123 def readPackageInfo(pkg, histfile):
124 pkginfo = PackageInfo(pkg)
125 with open(histfile, "r") as f:
126 for line in f:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500127 lns = line.split('=', 1)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500128 name = lns[0].strip()
129 value = lns[1].strip(" \t\r\n").strip('"')
130 if name == "PE":
131 pkginfo.pe = value
132 elif name == "PV":
133 pkginfo.pv = value
134 elif name == "PR":
135 pkginfo.pr = value
136 elif name == "PKG":
137 pkginfo.pkg = value
138 elif name == "PKGE":
139 pkginfo.pkge = value
140 elif name == "PKGV":
141 pkginfo.pkgv = value
142 elif name == "PKGR":
143 pkginfo.pkgr = value
144 elif name == "RPROVIDES":
145 pkginfo.rprovides = value
146 elif name == "RDEPENDS":
147 pkginfo.rdepends = value
148 elif name == "RRECOMMENDS":
149 pkginfo.rrecommends = value
150 elif name == "RSUGGESTS":
151 pkginfo.rsuggests = value
152 elif name == "RREPLACES":
153 pkginfo.rreplaces = value
154 elif name == "RCONFLICTS":
155 pkginfo.rconflicts = value
156 elif name == "PKGSIZE":
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600157 pkginfo.size = int(value)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500158 elif name == "FILES":
159 pkginfo.files = value
160 elif name == "FILELIST":
161 pkginfo.filelist = value
162 # Apply defaults
163 if not pkginfo.pkg:
164 pkginfo.pkg = pkginfo.name
165 if not pkginfo.pkge:
166 pkginfo.pkge = pkginfo.pe
167 if not pkginfo.pkgv:
168 pkginfo.pkgv = pkginfo.pv
169 if not pkginfo.pkgr:
170 pkginfo.pkgr = pkginfo.pr
171 return pkginfo
172
173 def getlastpkgversion(pkg):
174 try:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500175 histfile = os.path.join(oldpkghistdir, pkg, "latest")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500176 return readPackageInfo(pkg, histfile)
177 except EnvironmentError:
178 return None
179
180 def sortpkglist(string):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500181 pkgiter = re.finditer(r'[a-zA-Z0-9.+-]+( \([><=]+[^)]+\))?', string, 0)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500182 pkglist = [p.group(0) for p in pkgiter]
183 pkglist.sort()
184 return ' '.join(pkglist)
185
186 def sortlist(string):
187 items = string.split(' ')
188 items.sort()
189 return ' '.join(items)
190
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500191 pn = d.getVar('PN')
192 pe = d.getVar('PE') or "0"
193 pv = d.getVar('PV')
194 pr = d.getVar('PR')
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500195 layer = bb.utils.get_file_layer(d.getVar('FILE'), d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500196
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500197 pkgdata_dir = d.getVar('PKGDATA_DIR')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500198 packages = ""
199 try:
200 with open(os.path.join(pkgdata_dir, pn)) as f:
201 for line in f.readlines():
202 if line.startswith('PACKAGES: '):
203 packages = oe.utils.squashspaces(line.split(': ', 1)[1])
204 break
205 except IOError as e:
206 if e.errno == errno.ENOENT:
207 # Probably a -cross recipe, just ignore
208 return 0
209 else:
210 raise
211
212 packagelist = packages.split()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500213 preserve = d.getVar('BUILDHISTORY_PRESERVE').split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500214 if not os.path.exists(pkghistdir):
215 bb.utils.mkdirhier(pkghistdir)
216 else:
217 # Remove files for packages that no longer exist
218 for item in os.listdir(pkghistdir):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500219 if item not in preserve:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500220 if item not in packagelist:
221 itempath = os.path.join(pkghistdir, item)
222 if os.path.isdir(itempath):
223 for subfile in os.listdir(itempath):
224 os.unlink(os.path.join(itempath, subfile))
225 os.rmdir(itempath)
226 else:
227 os.unlink(itempath)
228
229 rcpinfo = RecipeInfo(pn)
230 rcpinfo.pe = pe
231 rcpinfo.pv = pv
232 rcpinfo.pr = pr
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500233 rcpinfo.depends = sortlist(oe.utils.squashspaces(d.getVar('DEPENDS') or ""))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500234 rcpinfo.packages = packages
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500235 rcpinfo.layer = layer
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500236 write_recipehistory(rcpinfo, d)
237
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500238 pkgdest = d.getVar('PKGDEST')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500239 for pkg in packagelist:
240 pkgdata = {}
241 with open(os.path.join(pkgdata_dir, 'runtime', pkg)) as f:
242 for line in f.readlines():
243 item = line.rstrip('\n').split(': ', 1)
244 key = item[0]
245 if key.endswith('_' + pkg):
246 key = key[:-len(pkg)-1]
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600247 pkgdata[key] = item[1]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500248
249 pkge = pkgdata.get('PKGE', '0')
250 pkgv = pkgdata['PKGV']
251 pkgr = pkgdata['PKGR']
252 #
253 # Find out what the last version was
254 # Make sure the version did not decrease
255 #
256 lastversion = getlastpkgversion(pkg)
257 if lastversion:
258 last_pkge = lastversion.pkge
259 last_pkgv = lastversion.pkgv
260 last_pkgr = lastversion.pkgr
261 r = bb.utils.vercmp((pkge, pkgv, pkgr), (last_pkge, last_pkgv, last_pkgr))
262 if r < 0:
263 msg = "Package version for package %s went backwards which would break package feeds from (%s:%s-%s to %s:%s-%s)" % (pkg, last_pkge, last_pkgv, last_pkgr, pkge, pkgv, pkgr)
264 package_qa_handle_error("version-going-backwards", msg, d)
265
266 pkginfo = PackageInfo(pkg)
267 # Apparently the version can be different on a per-package basis (see Python)
268 pkginfo.pe = pkgdata.get('PE', '0')
269 pkginfo.pv = pkgdata['PV']
270 pkginfo.pr = pkgdata['PR']
271 pkginfo.pkg = pkgdata['PKG']
272 pkginfo.pkge = pkge
273 pkginfo.pkgv = pkgv
274 pkginfo.pkgr = pkgr
275 pkginfo.rprovides = sortpkglist(oe.utils.squashspaces(pkgdata.get('RPROVIDES', "")))
276 pkginfo.rdepends = sortpkglist(oe.utils.squashspaces(pkgdata.get('RDEPENDS', "")))
277 pkginfo.rrecommends = sortpkglist(oe.utils.squashspaces(pkgdata.get('RRECOMMENDS', "")))
278 pkginfo.rsuggests = sortpkglist(oe.utils.squashspaces(pkgdata.get('RSUGGESTS', "")))
279 pkginfo.rreplaces = sortpkglist(oe.utils.squashspaces(pkgdata.get('RREPLACES', "")))
280 pkginfo.rconflicts = sortpkglist(oe.utils.squashspaces(pkgdata.get('RCONFLICTS', "")))
281 pkginfo.files = oe.utils.squashspaces(pkgdata.get('FILES', ""))
282 for filevar in pkginfo.filevars:
283 pkginfo.filevars[filevar] = pkgdata.get(filevar, "")
284
285 # Gather information about packaged files
286 val = pkgdata.get('FILES_INFO', '')
287 dictval = json.loads(val)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600288 filelist = list(dictval.keys())
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500289 filelist.sort()
290 pkginfo.filelist = " ".join(filelist)
291
292 pkginfo.size = int(pkgdata['PKGSIZE'])
293
294 write_pkghistory(pkginfo, d)
295
296 # Create files-in-<package-name>.txt files containing a list of files of each recipe's package
297 bb.build.exec_func("buildhistory_list_pkg_files", d)
298}
299
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500300python buildhistory_emit_outputsigs() {
301 if not "task" in (d.getVar('BUILDHISTORY_FEATURES') or "").split():
302 return
303
304 import hashlib
305
306 taskoutdir = os.path.join(d.getVar('BUILDHISTORY_DIR'), 'task', 'output')
307 bb.utils.mkdirhier(taskoutdir)
308 currenttask = d.getVar('BB_CURRENTTASK')
309 pn = d.getVar('PN')
310 taskfile = os.path.join(taskoutdir, '%s.%s' % (pn, currenttask))
311
312 cwd = os.getcwd()
313 filesigs = {}
314 for root, _, files in os.walk(cwd):
315 for fname in files:
316 if fname == 'fixmepath':
317 continue
318 fullpath = os.path.join(root, fname)
319 try:
320 if os.path.islink(fullpath):
321 sha256 = hashlib.sha256(os.readlink(fullpath).encode('utf-8')).hexdigest()
322 elif os.path.isfile(fullpath):
323 sha256 = bb.utils.sha256_file(fullpath)
324 else:
325 continue
326 except OSError:
327 bb.warn('buildhistory: unable to read %s to get output signature' % fullpath)
328 continue
329 filesigs[os.path.relpath(fullpath, cwd)] = sha256
330 with open(taskfile, 'w') as f:
331 for fpath, fsig in sorted(filesigs.items(), key=lambda item: item[0]):
332 f.write('%s %s\n' % (fpath, fsig))
333}
334
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500335
336def write_recipehistory(rcpinfo, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500337 bb.debug(2, "Writing recipe history")
338
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500339 pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500340
341 infofile = os.path.join(pkghistdir, "latest")
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600342 with open(infofile, "w") as f:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500343 if rcpinfo.pe != "0":
344 f.write(u"PE = %s\n" % rcpinfo.pe)
345 f.write(u"PV = %s\n" % rcpinfo.pv)
346 f.write(u"PR = %s\n" % rcpinfo.pr)
347 f.write(u"DEPENDS = %s\n" % rcpinfo.depends)
348 f.write(u"PACKAGES = %s\n" % rcpinfo.packages)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500349 f.write(u"LAYER = %s\n" % rcpinfo.layer)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500350
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500351 write_latest_srcrev(d, pkghistdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500352
353def write_pkghistory(pkginfo, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500354 bb.debug(2, "Writing package history for package %s" % pkginfo.name)
355
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500356 pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500357
358 pkgpath = os.path.join(pkghistdir, pkginfo.name)
359 if not os.path.exists(pkgpath):
360 bb.utils.mkdirhier(pkgpath)
361
362 infofile = os.path.join(pkgpath, "latest")
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600363 with open(infofile, "w") as f:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500364 if pkginfo.pe != "0":
365 f.write(u"PE = %s\n" % pkginfo.pe)
366 f.write(u"PV = %s\n" % pkginfo.pv)
367 f.write(u"PR = %s\n" % pkginfo.pr)
368
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600369 if pkginfo.pkg != pkginfo.name:
370 f.write(u"PKG = %s\n" % pkginfo.pkg)
371 if pkginfo.pkge != pkginfo.pe:
372 f.write(u"PKGE = %s\n" % pkginfo.pkge)
373 if pkginfo.pkgv != pkginfo.pv:
374 f.write(u"PKGV = %s\n" % pkginfo.pkgv)
375 if pkginfo.pkgr != pkginfo.pr:
376 f.write(u"PKGR = %s\n" % pkginfo.pkgr)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500377 f.write(u"RPROVIDES = %s\n" % pkginfo.rprovides)
378 f.write(u"RDEPENDS = %s\n" % pkginfo.rdepends)
379 f.write(u"RRECOMMENDS = %s\n" % pkginfo.rrecommends)
380 if pkginfo.rsuggests:
381 f.write(u"RSUGGESTS = %s\n" % pkginfo.rsuggests)
382 if pkginfo.rreplaces:
383 f.write(u"RREPLACES = %s\n" % pkginfo.rreplaces)
384 if pkginfo.rconflicts:
385 f.write(u"RCONFLICTS = %s\n" % pkginfo.rconflicts)
386 f.write(u"PKGSIZE = %d\n" % pkginfo.size)
387 f.write(u"FILES = %s\n" % pkginfo.files)
388 f.write(u"FILELIST = %s\n" % pkginfo.filelist)
389
390 for filevar in pkginfo.filevars:
391 filevarpath = os.path.join(pkgpath, "latest.%s" % filevar)
392 val = pkginfo.filevars[filevar]
393 if val:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600394 with open(filevarpath, "w") as f:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500395 f.write(val)
396 else:
397 if os.path.exists(filevarpath):
398 os.unlink(filevarpath)
399
400#
401# rootfs_type can be: image, sdk_target, sdk_host
402#
403def buildhistory_list_installed(d, rootfs_type="image"):
404 from oe.rootfs import image_list_installed_packages
405 from oe.sdk import sdk_list_installed_packages
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500406 from oe.utils import format_pkg_list
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500407
408 process_list = [('file', 'bh_installed_pkgs.txt'),\
409 ('deps', 'bh_installed_pkgs_deps.txt')]
410
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500411 if rootfs_type == "image":
412 pkgs = image_list_installed_packages(d)
413 else:
414 pkgs = sdk_list_installed_packages(d, rootfs_type == "sdk_target")
415
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500416 for output_type, output_file in process_list:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500417 output_file_full = os.path.join(d.getVar('WORKDIR'), output_file)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500418
419 with open(output_file_full, 'w') as output:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500420 output.write(format_pkg_list(pkgs, output_type))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500421
422python buildhistory_list_installed_image() {
423 buildhistory_list_installed(d)
424}
425
426python buildhistory_list_installed_sdk_target() {
427 buildhistory_list_installed(d, "sdk_target")
428}
429
430python buildhistory_list_installed_sdk_host() {
431 buildhistory_list_installed(d, "sdk_host")
432}
433
434buildhistory_get_installed() {
435 mkdir -p $1
436
437 # Get list of installed packages
438 pkgcache="$1/installed-packages.tmp"
439 cat ${WORKDIR}/bh_installed_pkgs.txt | sort > $pkgcache && rm ${WORKDIR}/bh_installed_pkgs.txt
440
441 cat $pkgcache | awk '{ print $1 }' > $1/installed-package-names.txt
442 if [ -s $pkgcache ] ; then
443 cat $pkgcache | awk '{ print $2 }' | xargs -n1 basename > $1/installed-packages.txt
444 else
445 printf "" > $1/installed-packages.txt
446 fi
447
448 # Produce dependency graph
449 # First, quote each name to handle characters that cause issues for dot
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500450 sed 's:\([^| ]*\):"\1":g' ${WORKDIR}/bh_installed_pkgs_deps.txt > $1/depends.tmp &&
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500451 rm ${WORKDIR}/bh_installed_pkgs_deps.txt
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500452 # Remove lines with rpmlib(...) and config(...) dependencies, change the
453 # delimiter from pipe to "->", set the style for recommend lines and
454 # turn versioned dependencies into edge labels.
455 sed -i -e '/rpmlib(/d' \
456 -e '/config(/d' \
457 -e 's:|: -> :' \
458 -e 's:"\[REC\]":[style=dotted]:' \
459 -e 's:"\([<>=]\+\)" "\([^"]*\)":[label="\1 \2"]:' \
460 $1/depends.tmp
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500461 # Add header, sorted and de-duped contents and footer and then delete the temp file
462 printf "digraph depends {\n node [shape=plaintext]\n" > $1/depends.dot
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500463 cat $1/depends.tmp | sort -u >> $1/depends.dot
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500464 echo "}" >> $1/depends.dot
465 rm $1/depends.tmp
466
467 # Produce installed package sizes list
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500468 oe-pkgdata-util -p ${PKGDATA_DIR} read-value "PKGSIZE" -n -f $pkgcache > $1/installed-package-sizes.tmp
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500469 cat $1/installed-package-sizes.tmp | awk '{print $2 "\tKiB\t" $1}' | sort -n -r > $1/installed-package-sizes.txt
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500470 rm $1/installed-package-sizes.tmp
471
472 # We're now done with the cache, delete it
473 rm $pkgcache
474
475 if [ "$2" != "sdk" ] ; then
476 # Produce some cut-down graphs (for readability)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500477 grep -v kernel-image $1/depends.dot | grep -v kernel-3 | grep -v kernel-4 > $1/depends-nokernel.dot
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500478 grep -v libc6 $1/depends-nokernel.dot | grep -v libgcc > $1/depends-nokernel-nolibc.dot
479 grep -v update- $1/depends-nokernel-nolibc.dot > $1/depends-nokernel-nolibc-noupdate.dot
480 grep -v kernel-module $1/depends-nokernel-nolibc-noupdate.dot > $1/depends-nokernel-nolibc-noupdate-nomodules.dot
481 fi
482
483 # add complementary package information
484 if [ -e ${WORKDIR}/complementary_pkgs.txt ]; then
485 cp ${WORKDIR}/complementary_pkgs.txt $1
486 fi
487}
488
489buildhistory_get_image_installed() {
490 # Anything requiring the use of the packaging system should be done in here
491 # in case the packaging files are going to be removed for this image
492
493 if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'image', '1', '0', d)}" = "0" ] ; then
494 return
495 fi
496
497 buildhistory_get_installed ${BUILDHISTORY_DIR_IMAGE}
498}
499
500buildhistory_get_sdk_installed() {
501 # Anything requiring the use of the packaging system should be done in here
502 # in case the packaging files are going to be removed for this SDK
503
504 if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'sdk', '1', '0', d)}" = "0" ] ; then
505 return
506 fi
507
508 buildhistory_get_installed ${BUILDHISTORY_DIR_SDK}/$1 sdk
509}
510
511buildhistory_get_sdk_installed_host() {
512 buildhistory_get_sdk_installed host
513}
514
515buildhistory_get_sdk_installed_target() {
516 buildhistory_get_sdk_installed target
517}
518
519buildhistory_list_files() {
520 # List the files in the specified directory, but exclude date/time etc.
521 # This awk script is somewhat messy, but handles where the size is not printed for device files under pseudo
522 if [ "$3" = "fakeroot" ] ; then
523 ( cd $1 && ${FAKEROOTENV} ${FAKEROOTCMD} find . ! -path . -printf "%M %-10u %-10g %10s %p -> %l\n" | sort -k5 | sed 's/ * -> $//' > $2 )
524 else
525 ( cd $1 && find . ! -path . -printf "%M %-10u %-10g %10s %p -> %l\n" | sort -k5 | sed 's/ * -> $//' > $2 )
526 fi
527}
528
529buildhistory_list_pkg_files() {
530 # Create individual files-in-package for each recipe's package
531 for pkgdir in $(find ${PKGDEST}/* -maxdepth 0 -type d); do
532 pkgname=$(basename $pkgdir)
533 outfolder="${BUILDHISTORY_DIR_PACKAGE}/$pkgname"
534 outfile="$outfolder/files-in-package.txt"
535 # Make sure the output folder exists so we can create the file
536 if [ ! -d $outfolder ] ; then
537 bbdebug 2 "Folder $outfolder does not exist, file $outfile not created"
538 continue
539 fi
540 buildhistory_list_files $pkgdir $outfile fakeroot
541 done
542}
543
544buildhistory_get_imageinfo() {
545 if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'image', '1', '0', d)}" = "0" ] ; then
546 return
547 fi
548
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500549 mkdir -p ${BUILDHISTORY_DIR_IMAGE}
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500550 buildhistory_list_files ${IMAGE_ROOTFS} ${BUILDHISTORY_DIR_IMAGE}/files-in-image.txt
551
552 # Collect files requested in BUILDHISTORY_IMAGE_FILES
553 rm -rf ${BUILDHISTORY_DIR_IMAGE}/image-files
554 for f in ${BUILDHISTORY_IMAGE_FILES}; do
555 if [ -f ${IMAGE_ROOTFS}/$f ] ; then
556 mkdir -p ${BUILDHISTORY_DIR_IMAGE}/image-files/`dirname $f`
557 cp ${IMAGE_ROOTFS}/$f ${BUILDHISTORY_DIR_IMAGE}/image-files/$f
558 fi
559 done
560
561 # Record some machine-readable meta-information about the image
562 printf "" > ${BUILDHISTORY_DIR_IMAGE}/image-info.txt
563 cat >> ${BUILDHISTORY_DIR_IMAGE}/image-info.txt <<END
564${@buildhistory_get_imagevars(d)}
565END
566 imagesize=`du -ks ${IMAGE_ROOTFS} | awk '{ print $1 }'`
567 echo "IMAGESIZE = $imagesize" >> ${BUILDHISTORY_DIR_IMAGE}/image-info.txt
568
569 # Add some configuration information
570 echo "${MACHINE}: ${IMAGE_BASENAME} configured for ${DISTRO} ${DISTRO_VERSION}" > ${BUILDHISTORY_DIR_IMAGE}/build-id.txt
571
572 cat >> ${BUILDHISTORY_DIR_IMAGE}/build-id.txt <<END
573${@buildhistory_get_build_id(d)}
574END
575}
576
577buildhistory_get_sdkinfo() {
578 if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'sdk', '1', '0', d)}" = "0" ] ; then
579 return
580 fi
581
582 buildhistory_list_files ${SDK_OUTPUT} ${BUILDHISTORY_DIR_SDK}/files-in-sdk.txt
583
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500584 # Collect files requested in BUILDHISTORY_SDK_FILES
585 rm -rf ${BUILDHISTORY_DIR_SDK}/sdk-files
586 for f in ${BUILDHISTORY_SDK_FILES}; do
587 if [ -f ${SDK_OUTPUT}/${SDKPATH}/$f ] ; then
588 mkdir -p ${BUILDHISTORY_DIR_SDK}/sdk-files/`dirname $f`
589 cp ${SDK_OUTPUT}/${SDKPATH}/$f ${BUILDHISTORY_DIR_SDK}/sdk-files/$f
590 fi
591 done
592
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500593 # Record some machine-readable meta-information about the SDK
594 printf "" > ${BUILDHISTORY_DIR_SDK}/sdk-info.txt
595 cat >> ${BUILDHISTORY_DIR_SDK}/sdk-info.txt <<END
596${@buildhistory_get_sdkvars(d)}
597END
598 sdksize=`du -ks ${SDK_OUTPUT} | awk '{ print $1 }'`
599 echo "SDKSIZE = $sdksize" >> ${BUILDHISTORY_DIR_SDK}/sdk-info.txt
600}
601
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500602python buildhistory_get_extra_sdkinfo() {
603 import operator
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500604 from oe.sdk import get_extra_sdkinfo
605
606 sstate_dir = d.expand('${SDK_OUTPUT}/${SDKPATH}/sstate-cache')
607 extra_info = get_extra_sdkinfo(sstate_dir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500608
609 if d.getVar('BB_CURRENTTASK') == 'populate_sdk_ext' and \
610 "sdk" in (d.getVar('BUILDHISTORY_FEATURES') or "").split():
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500611 with open(d.expand('${BUILDHISTORY_DIR_SDK}/sstate-package-sizes.txt'), 'w') as f:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500612 filesizes_sorted = sorted(extra_info['filesizes'].items(), key=operator.itemgetter(1, 0), reverse=True)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500613 for fn, size in filesizes_sorted:
614 f.write('%10d KiB %s\n' % (size, fn))
615 with open(d.expand('${BUILDHISTORY_DIR_SDK}/sstate-task-sizes.txt'), 'w') as f:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500616 tasksizes_sorted = sorted(extra_info['tasksizes'].items(), key=operator.itemgetter(1, 0), reverse=True)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500617 for task, size in tasksizes_sorted:
618 f.write('%10d KiB %s\n' % (size, task))
619}
620
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500621# By using ROOTFS_POSTUNINSTALL_COMMAND we get in after uninstallation of
622# unneeded packages but before the removal of packaging files
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500623ROOTFS_POSTUNINSTALL_COMMAND += "buildhistory_list_installed_image ;"
624ROOTFS_POSTUNINSTALL_COMMAND += "buildhistory_get_image_installed ;"
625ROOTFS_POSTUNINSTALL_COMMAND[vardepvalueexclude] .= "| buildhistory_list_installed_image ;| buildhistory_get_image_installed ;"
626ROOTFS_POSTUNINSTALL_COMMAND[vardepsexclude] += "buildhistory_list_installed_image buildhistory_get_image_installed"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500627
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500628IMAGE_POSTPROCESS_COMMAND += "buildhistory_get_imageinfo ;"
629IMAGE_POSTPROCESS_COMMAND[vardepvalueexclude] .= "| buildhistory_get_imageinfo ;"
630IMAGE_POSTPROCESS_COMMAND[vardepsexclude] += "buildhistory_get_imageinfo"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500631
632# We want these to be the last run so that we get called after complementary package installation
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500633POPULATE_SDK_POST_TARGET_COMMAND_append = " buildhistory_list_installed_sdk_target;"
634POPULATE_SDK_POST_TARGET_COMMAND_append = " buildhistory_get_sdk_installed_target;"
635POPULATE_SDK_POST_TARGET_COMMAND[vardepvalueexclude] .= "| buildhistory_list_installed_sdk_target;| buildhistory_get_sdk_installed_target;"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500636
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500637POPULATE_SDK_POST_HOST_COMMAND_append = " buildhistory_list_installed_sdk_host;"
638POPULATE_SDK_POST_HOST_COMMAND_append = " buildhistory_get_sdk_installed_host;"
639POPULATE_SDK_POST_HOST_COMMAND[vardepvalueexclude] .= "| buildhistory_list_installed_sdk_host;| buildhistory_get_sdk_installed_host;"
640
641SDK_POSTPROCESS_COMMAND_append = " buildhistory_get_sdkinfo ; buildhistory_get_extra_sdkinfo; "
642SDK_POSTPROCESS_COMMAND[vardepvalueexclude] .= "| buildhistory_get_sdkinfo ; buildhistory_get_extra_sdkinfo; "
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500643
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500644python buildhistory_write_sigs() {
645 if not "task" in (d.getVar('BUILDHISTORY_FEATURES') or "").split():
646 return
647
648 # Create sigs file
649 if hasattr(bb.parse.siggen, 'dump_siglist'):
650 taskoutdir = os.path.join(d.getVar('BUILDHISTORY_DIR'), 'task')
651 bb.utils.mkdirhier(taskoutdir)
652 bb.parse.siggen.dump_siglist(os.path.join(taskoutdir, 'tasksigs.txt'))
653}
654
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500655def buildhistory_get_build_id(d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500656 if d.getVar('BB_WORKERCONTEXT') != '1':
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500657 return ""
658 localdata = bb.data.createCopy(d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500659 statuslines = []
660 for func in oe.data.typed_value('BUILDCFG_FUNCS', localdata):
661 g = globals()
662 if func not in g:
663 bb.warn("Build configuration function '%s' does not exist" % func)
664 else:
665 flines = g[func](localdata)
666 if flines:
667 statuslines.extend(flines)
668
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500669 statusheader = d.getVar('BUILDCFG_HEADER')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500670 return('\n%s\n%s\n' % (statusheader, '\n'.join(statuslines)))
671
672def buildhistory_get_metadata_revs(d):
673 # We want an easily machine-readable format here, so get_layers_branch_rev isn't quite what we want
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500674 layers = (d.getVar("BBLAYERS") or "").split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500675 medadata_revs = ["%-17s = %s:%s" % (os.path.basename(i), \
676 base_get_metadata_git_branch(i, None).strip(), \
677 base_get_metadata_git_revision(i, None)) \
678 for i in layers]
679 return '\n'.join(medadata_revs)
680
681def outputvars(vars, listvars, d):
682 vars = vars.split()
683 listvars = listvars.split()
684 ret = ""
685 for var in vars:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500686 value = d.getVar(var) or ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500687 if var in listvars:
688 # Squash out spaces
689 value = oe.utils.squashspaces(value)
690 ret += "%s = %s\n" % (var, value)
691 return ret.rstrip('\n')
692
693def buildhistory_get_imagevars(d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500694 if d.getVar('BB_WORKERCONTEXT') != '1':
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500695 return ""
696 imagevars = "DISTRO DISTRO_VERSION USER_CLASSES IMAGE_CLASSES IMAGE_FEATURES IMAGE_LINGUAS IMAGE_INSTALL BAD_RECOMMENDATIONS NO_RECOMMENDATIONS PACKAGE_EXCLUDE ROOTFS_POSTPROCESS_COMMAND IMAGE_POSTPROCESS_COMMAND"
697 listvars = "USER_CLASSES IMAGE_CLASSES IMAGE_FEATURES IMAGE_LINGUAS IMAGE_INSTALL BAD_RECOMMENDATIONS PACKAGE_EXCLUDE"
698 return outputvars(imagevars, listvars, d)
699
700def buildhistory_get_sdkvars(d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500701 if d.getVar('BB_WORKERCONTEXT') != '1':
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500702 return ""
703 sdkvars = "DISTRO DISTRO_VERSION SDK_NAME SDK_VERSION SDKMACHINE SDKIMAGE_FEATURES BAD_RECOMMENDATIONS NO_RECOMMENDATIONS PACKAGE_EXCLUDE"
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500704 if d.getVar('BB_CURRENTTASK') == 'populate_sdk_ext':
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500705 # Extensible SDK uses some additional variables
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600706 sdkvars += " SDK_LOCAL_CONF_WHITELIST SDK_LOCAL_CONF_BLACKLIST SDK_INHERIT_BLACKLIST SDK_UPDATE_URL SDK_EXT_TYPE SDK_RECRDEP_TASKS SDK_INCLUDE_PKGDATA SDK_INCLUDE_TOOLCHAIN"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500707 listvars = "SDKIMAGE_FEATURES BAD_RECOMMENDATIONS PACKAGE_EXCLUDE SDK_LOCAL_CONF_WHITELIST SDK_LOCAL_CONF_BLACKLIST SDK_INHERIT_BLACKLIST"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500708 return outputvars(sdkvars, listvars, d)
709
710
711def buildhistory_get_cmdline(d):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500712 argv = d.getVar('BB_CMDLINE', False)
713 if argv:
714 if argv[0].endswith('bin/bitbake'):
715 bincmd = 'bitbake'
716 else:
717 bincmd = argv[0]
718 return '%s %s' % (bincmd, ' '.join(argv[1:]))
719 return ''
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500720
721
722buildhistory_single_commit() {
723 if [ "$3" = "" ] ; then
724 commitopts="${BUILDHISTORY_DIR}/ --allow-empty"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500725 shortlogprefix="No changes: "
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500726 else
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500727 commitopts=""
728 shortlogprefix=""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500729 fi
730 if [ "${BUILDHISTORY_BUILD_FAILURES}" = "0" ] ; then
731 result="succeeded"
732 else
733 result="failed"
734 fi
735 case ${BUILDHISTORY_BUILD_INTERRUPTED} in
736 1)
737 result="$result (interrupted)"
738 ;;
739 2)
740 result="$result (force interrupted)"
741 ;;
742 esac
743 commitmsgfile=`mktemp`
744 cat > $commitmsgfile << END
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500745${shortlogprefix}Build ${BUILDNAME} of ${DISTRO} ${DISTRO_VERSION} for machine ${MACHINE} on $2
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500746
747cmd: $1
748
749result: $result
750
751metadata revisions:
752END
753 cat ${BUILDHISTORY_DIR}/metadata-revs >> $commitmsgfile
754 git commit $commitopts -F $commitmsgfile --author "${BUILDHISTORY_COMMIT_AUTHOR}" > /dev/null
755 rm $commitmsgfile
756}
757
758buildhistory_commit() {
759 if [ ! -d ${BUILDHISTORY_DIR} ] ; then
760 # Code above that creates this dir never executed, so there can't be anything to commit
761 return
762 fi
763
764 # Create a machine-readable list of metadata revisions for each layer
765 cat > ${BUILDHISTORY_DIR}/metadata-revs <<END
766${@buildhistory_get_metadata_revs(d)}
767END
768
769 ( cd ${BUILDHISTORY_DIR}/
770 # Initialise the repo if necessary
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500771 if [ ! -e .git ] ; then
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500772 git init -q
773 else
774 git tag -f build-minus-3 build-minus-2 > /dev/null 2>&1 || true
775 git tag -f build-minus-2 build-minus-1 > /dev/null 2>&1 || true
776 git tag -f build-minus-1 > /dev/null 2>&1 || true
777 fi
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600778
779 check_git_config
780
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500781 # Check if there are new/changed files to commit (other than metadata-revs)
782 repostatus=`git status --porcelain | grep -v " metadata-revs$"`
783 HOSTNAME=`hostname 2>/dev/null || echo unknown`
784 CMDLINE="${@buildhistory_get_cmdline(d)}"
785 if [ "$repostatus" != "" ] ; then
786 git add -A .
787 # porcelain output looks like "?? packages/foo/bar"
788 # Ensure we commit metadata-revs with the first commit
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500789 buildhistory_single_commit "$CMDLINE" "$HOSTNAME" dummy
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500790 git gc --auto --quiet
791 else
792 buildhistory_single_commit "$CMDLINE" "$HOSTNAME"
793 fi
794 if [ "${BUILDHISTORY_PUSH_REPO}" != "" ] ; then
795 git push -q ${BUILDHISTORY_PUSH_REPO}
796 fi) || true
797}
798
799python buildhistory_eventhandler() {
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500800 if e.data.getVar('BUILDHISTORY_FEATURES').strip():
801 reset = e.data.getVar("BUILDHISTORY_RESET")
802 olddir = e.data.getVar("BUILDHISTORY_OLD_DIR")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500803 if isinstance(e, bb.event.BuildStarted):
804 if reset:
805 import shutil
806 # Clean up after potentially interrupted build.
807 if os.path.isdir(olddir):
808 shutil.rmtree(olddir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500809 rootdir = e.data.getVar("BUILDHISTORY_DIR")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500810 entries = [ x for x in os.listdir(rootdir) if not x.startswith('.') ]
811 bb.utils.mkdirhier(olddir)
812 for entry in entries:
813 os.rename(os.path.join(rootdir, entry),
814 os.path.join(olddir, entry))
815 elif isinstance(e, bb.event.BuildCompleted):
816 if reset:
817 import shutil
818 shutil.rmtree(olddir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500819 if e.data.getVar("BUILDHISTORY_COMMIT") == "1":
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500820 bb.note("Writing buildhistory")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500821 bb.build.exec_func("buildhistory_write_sigs", d)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500822 localdata = bb.data.createCopy(e.data)
823 localdata.setVar('BUILDHISTORY_BUILD_FAILURES', str(e._failures))
824 interrupted = getattr(e, '_interrupted', 0)
825 localdata.setVar('BUILDHISTORY_BUILD_INTERRUPTED', str(interrupted))
826 bb.build.exec_func("buildhistory_commit", localdata)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500827 else:
828 bb.note("No commit since BUILDHISTORY_COMMIT != '1'")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500829}
830
831addhandler buildhistory_eventhandler
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500832buildhistory_eventhandler[eventmask] = "bb.event.BuildCompleted bb.event.BuildStarted"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500833
834
835# FIXME this ought to be moved into the fetcher
836def _get_srcrev_values(d):
837 """
838 Return the version strings for the current recipe
839 """
840
841 scms = []
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500842 fetcher = bb.fetch.Fetch(d.getVar('SRC_URI').split(), d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500843 urldata = fetcher.ud
844 for u in urldata:
845 if urldata[u].method.supports_srcrev():
846 scms.append(u)
847
848 autoinc_templ = 'AUTOINC+'
849 dict_srcrevs = {}
850 dict_tag_srcrevs = {}
851 for scm in scms:
852 ud = urldata[scm]
853 for name in ud.names:
854 try:
855 rev = ud.method.sortable_revision(ud, d, name)
856 except TypeError:
857 # support old bitbake versions
858 rev = ud.method.sortable_revision(scm, ud, d, name)
859 # Clean this up when we next bump bitbake version
860 if type(rev) != str:
861 autoinc, rev = rev
862 elif rev.startswith(autoinc_templ):
863 rev = rev[len(autoinc_templ):]
864 dict_srcrevs[name] = rev
865 if 'tag' in ud.parm:
866 tag = ud.parm['tag'];
867 key = name+'_'+tag
868 dict_tag_srcrevs[key] = rev
869 return (dict_srcrevs, dict_tag_srcrevs)
870
871do_fetch[postfuncs] += "write_srcrev"
872do_fetch[vardepsexclude] += "write_srcrev"
873python write_srcrev() {
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500874 write_latest_srcrev(d, d.getVar('BUILDHISTORY_DIR_PACKAGE'))
875}
876
877def write_latest_srcrev(d, pkghistdir):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500878 srcrevfile = os.path.join(pkghistdir, 'latest_srcrev')
879
880 srcrevs, tag_srcrevs = _get_srcrev_values(d)
881 if srcrevs:
882 if not os.path.exists(pkghistdir):
883 bb.utils.mkdirhier(pkghistdir)
884 old_tag_srcrevs = {}
885 if os.path.exists(srcrevfile):
886 with open(srcrevfile) as f:
887 for line in f:
888 if line.startswith('# tag_'):
889 key, value = line.split("=", 1)
890 key = key.replace('# tag_', '').strip()
891 value = value.replace('"', '').strip()
892 old_tag_srcrevs[key] = value
893 with open(srcrevfile, 'w') as f:
894 orig_srcrev = d.getVar('SRCREV', False) or 'INVALID'
895 if orig_srcrev != 'INVALID':
896 f.write('# SRCREV = "%s"\n' % orig_srcrev)
897 if len(srcrevs) > 1:
898 for name, srcrev in srcrevs.items():
899 orig_srcrev = d.getVar('SRCREV_%s' % name, False)
900 if orig_srcrev:
901 f.write('# SRCREV_%s = "%s"\n' % (name, orig_srcrev))
902 f.write('SRCREV_%s = "%s"\n' % (name, srcrev))
903 else:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500904 f.write('SRCREV = "%s"\n' % next(iter(srcrevs.values())))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500905 if len(tag_srcrevs) > 0:
906 for name, srcrev in tag_srcrevs.items():
907 f.write('# tag_%s = "%s"\n' % (name, srcrev))
908 if name in old_tag_srcrevs and old_tag_srcrevs[name] != srcrev:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500909 pkg = d.getVar('PN')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500910 bb.warn("Revision for tag %s in package %s was changed since last build (from %s to %s)" % (name, pkg, old_tag_srcrevs[name], srcrev))
911
912 else:
913 if os.path.exists(srcrevfile):
914 os.remove(srcrevfile)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500915
916do_testimage[postfuncs] += "write_ptest_result"
917do_testimage[vardepsexclude] += "write_ptest_result"
918
919python write_ptest_result() {
920 write_latest_ptest_result(d, d.getVar('BUILDHISTORY_DIR'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500921}
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500922
923def write_latest_ptest_result(d, histdir):
924 import glob
925 import subprocess
926 test_log_dir = d.getVar('TEST_LOG_DIR')
927 input_ptest = os.path.join(test_log_dir, 'ptest_log')
928 output_ptest = os.path.join(histdir, 'ptest')
929 if os.path.exists(input_ptest):
930 try:
931 # Lock it avoid race issue
932 lock = bb.utils.lockfile(output_ptest + "/ptest.lock")
933 bb.utils.mkdirhier(output_ptest)
934 oe.path.copytree(input_ptest, output_ptest)
935 # Sort test result
936 for result in glob.glob('%s/pass.fail.*' % output_ptest):
937 bb.debug(1, 'Processing %s' % result)
938 cmd = ['sort', result, '-o', result]
939 bb.debug(1, 'Running %s' % cmd)
940 ret = subprocess.call(cmd)
941 if ret != 0:
942 bb.error('Failed to run %s!' % cmd)
943 finally:
944 bb.utils.unlockfile(lock)