| # |
| # Records history of build output in order to detect regressions |
| # |
| # Based in part on testlab.bbclass and packagehistory.bbclass |
| # |
| # Copyright (C) 2011-2016 Intel Corporation |
| # Copyright (C) 2007-2011 Koen Kooi <koen@openembedded.org> |
| # |
| |
| BUILDHISTORY_FEATURES ?= "image package sdk" |
| BUILDHISTORY_DIR ?= "${TOPDIR}/buildhistory" |
| BUILDHISTORY_DIR_IMAGE = "${BUILDHISTORY_DIR}/images/${MACHINE_ARCH}/${TCLIBC}/${IMAGE_BASENAME}" |
| BUILDHISTORY_DIR_PACKAGE = "${BUILDHISTORY_DIR}/packages/${MULTIMACH_TARGET_SYS}/${PN}" |
| |
| # Setting this to non-empty will remove the old content of the buildhistory as part of |
| # the current bitbake invocation and replace it with information about what was built |
| # during the build. |
| # |
| # This is meant to be used in continuous integration (CI) systems when invoking bitbake |
| # for full world builds. The effect in that case is that information about packages |
| # that no longer get build also gets removed from the buildhistory, which is not |
| # the case otherwise. |
| # |
| # The advantage over manually cleaning the buildhistory outside of bitbake is that |
| # the "version-going-backwards" check still works. When relying on that, be careful |
| # about failed world builds: they will lead to incomplete information in the |
| # buildhistory because information about packages that could not be built will |
| # also get removed. A CI system should handle that by discarding the buildhistory |
| # of failed builds. |
| # |
| # The expected usage is via auto.conf, but passing via the command line also works |
| # with: BB_ENV_EXTRAWHITE=BUILDHISTORY_RESET BUILDHISTORY_RESET=1 |
| BUILDHISTORY_RESET ?= "" |
| |
| BUILDHISTORY_OLD_DIR = "${BUILDHISTORY_DIR}/${@ "old" if "${BUILDHISTORY_RESET}" else ""}" |
| BUILDHISTORY_OLD_DIR_PACKAGE = "${BUILDHISTORY_OLD_DIR}/packages/${MULTIMACH_TARGET_SYS}/${PN}" |
| BUILDHISTORY_DIR_SDK = "${BUILDHISTORY_DIR}/sdk/${SDK_NAME}${SDK_EXT}/${IMAGE_BASENAME}" |
| BUILDHISTORY_IMAGE_FILES ?= "/etc/passwd /etc/group" |
| BUILDHISTORY_SDK_FILES ?= "conf/local.conf conf/bblayers.conf conf/auto.conf conf/locked-sigs.inc conf/devtool.conf" |
| BUILDHISTORY_COMMIT ?= "0" |
| BUILDHISTORY_COMMIT_AUTHOR ?= "buildhistory <buildhistory@${DISTRO}>" |
| BUILDHISTORY_PUSH_REPO ?= "" |
| |
| SSTATEPOSTINSTFUNCS_append = " buildhistory_emit_pkghistory" |
| # We want to avoid influencing the signatures of sstate tasks - first the function itself: |
| sstate_install[vardepsexclude] += "buildhistory_emit_pkghistory" |
| # then the value added to SSTATEPOSTINSTFUNCS: |
| SSTATEPOSTINSTFUNCS[vardepvalueexclude] .= "| buildhistory_emit_pkghistory" |
| |
| # Similarly for our function that gets the output signatures |
| SSTATEPOSTUNPACKFUNCS_append = " buildhistory_emit_outputsigs" |
| sstate_installpkgdir[vardepsexclude] += "buildhistory_emit_outputsigs" |
| SSTATEPOSTUNPACKFUNCS[vardepvalueexclude] .= "| buildhistory_emit_outputsigs" |
| |
| # All items excepts those listed here will be removed from a recipe's |
| # build history directory by buildhistory_emit_pkghistory(). This is |
| # necessary because some of these items (package directories, files that |
| # we no longer emit) might be obsolete. |
| # |
| # When extending build history, derive your class from buildhistory.bbclass |
| # and extend this list here with the additional files created by the derived |
| # class. |
| BUILDHISTORY_PRESERVE = "latest latest_srcrev" |
| |
| PATCH_GIT_USER_EMAIL ?= "buildhistory@oe" |
| PATCH_GIT_USER_NAME ?= "OpenEmbedded" |
| |
| # |
| # Write out metadata about this package for comparison when writing future packages |
| # |
| python buildhistory_emit_pkghistory() { |
| if not d.getVar('BB_CURRENTTASK') in ['packagedata', 'packagedata_setscene']: |
| return 0 |
| |
| if not "package" in (d.getVar('BUILDHISTORY_FEATURES') or "").split(): |
| return 0 |
| |
| import re |
| import json |
| import errno |
| |
| pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE') |
| oldpkghistdir = d.getVar('BUILDHISTORY_OLD_DIR_PACKAGE') |
| |
| class RecipeInfo: |
| def __init__(self, name): |
| self.name = name |
| self.pe = "0" |
| self.pv = "0" |
| self.pr = "r0" |
| self.depends = "" |
| self.packages = "" |
| self.srcrev = "" |
| self.layer = "" |
| |
| |
| class PackageInfo: |
| def __init__(self, name): |
| self.name = name |
| self.pe = "0" |
| self.pv = "0" |
| self.pr = "r0" |
| # pkg/pkge/pkgv/pkgr should be empty because we want to be able to default them |
| self.pkg = "" |
| self.pkge = "" |
| self.pkgv = "" |
| self.pkgr = "" |
| self.size = 0 |
| self.depends = "" |
| self.rprovides = "" |
| self.rdepends = "" |
| self.rrecommends = "" |
| self.rsuggests = "" |
| self.rreplaces = "" |
| self.rconflicts = "" |
| self.files = "" |
| self.filelist = "" |
| # Variables that need to be written to their own separate file |
| self.filevars = dict.fromkeys(['pkg_preinst', 'pkg_postinst', 'pkg_prerm', 'pkg_postrm']) |
| |
| # Should check PACKAGES here to see if anything removed |
| |
| def readPackageInfo(pkg, histfile): |
| pkginfo = PackageInfo(pkg) |
| with open(histfile, "r") as f: |
| for line in f: |
| lns = line.split('=', 1) |
| name = lns[0].strip() |
| value = lns[1].strip(" \t\r\n").strip('"') |
| if name == "PE": |
| pkginfo.pe = value |
| elif name == "PV": |
| pkginfo.pv = value |
| elif name == "PR": |
| pkginfo.pr = value |
| elif name == "PKG": |
| pkginfo.pkg = value |
| elif name == "PKGE": |
| pkginfo.pkge = value |
| elif name == "PKGV": |
| pkginfo.pkgv = value |
| elif name == "PKGR": |
| pkginfo.pkgr = value |
| elif name == "RPROVIDES": |
| pkginfo.rprovides = value |
| elif name == "RDEPENDS": |
| pkginfo.rdepends = value |
| elif name == "RRECOMMENDS": |
| pkginfo.rrecommends = value |
| elif name == "RSUGGESTS": |
| pkginfo.rsuggests = value |
| elif name == "RREPLACES": |
| pkginfo.rreplaces = value |
| elif name == "RCONFLICTS": |
| pkginfo.rconflicts = value |
| elif name == "PKGSIZE": |
| pkginfo.size = int(value) |
| elif name == "FILES": |
| pkginfo.files = value |
| elif name == "FILELIST": |
| pkginfo.filelist = value |
| # Apply defaults |
| if not pkginfo.pkg: |
| pkginfo.pkg = pkginfo.name |
| if not pkginfo.pkge: |
| pkginfo.pkge = pkginfo.pe |
| if not pkginfo.pkgv: |
| pkginfo.pkgv = pkginfo.pv |
| if not pkginfo.pkgr: |
| pkginfo.pkgr = pkginfo.pr |
| return pkginfo |
| |
| def getlastpkgversion(pkg): |
| try: |
| histfile = os.path.join(oldpkghistdir, pkg, "latest") |
| return readPackageInfo(pkg, histfile) |
| except EnvironmentError: |
| return None |
| |
| def sortpkglist(string): |
| pkgiter = re.finditer(r'[a-zA-Z0-9.+-]+( \([><=]+[^)]+\))?', string, 0) |
| pkglist = [p.group(0) for p in pkgiter] |
| pkglist.sort() |
| return ' '.join(pkglist) |
| |
| def sortlist(string): |
| items = string.split(' ') |
| items.sort() |
| return ' '.join(items) |
| |
| pn = d.getVar('PN') |
| pe = d.getVar('PE') or "0" |
| pv = d.getVar('PV') |
| pr = d.getVar('PR') |
| layer = bb.utils.get_file_layer(d.getVar('FILE'), d) |
| |
| pkgdata_dir = d.getVar('PKGDATA_DIR') |
| packages = "" |
| try: |
| with open(os.path.join(pkgdata_dir, pn)) as f: |
| for line in f.readlines(): |
| if line.startswith('PACKAGES: '): |
| packages = oe.utils.squashspaces(line.split(': ', 1)[1]) |
| break |
| except IOError as e: |
| if e.errno == errno.ENOENT: |
| # Probably a -cross recipe, just ignore |
| return 0 |
| else: |
| raise |
| |
| packagelist = packages.split() |
| preserve = d.getVar('BUILDHISTORY_PRESERVE').split() |
| if not os.path.exists(pkghistdir): |
| bb.utils.mkdirhier(pkghistdir) |
| else: |
| # Remove files for packages that no longer exist |
| for item in os.listdir(pkghistdir): |
| if item not in preserve: |
| if item not in packagelist: |
| itempath = os.path.join(pkghistdir, item) |
| if os.path.isdir(itempath): |
| for subfile in os.listdir(itempath): |
| os.unlink(os.path.join(itempath, subfile)) |
| os.rmdir(itempath) |
| else: |
| os.unlink(itempath) |
| |
| rcpinfo = RecipeInfo(pn) |
| rcpinfo.pe = pe |
| rcpinfo.pv = pv |
| rcpinfo.pr = pr |
| rcpinfo.depends = sortlist(oe.utils.squashspaces(d.getVar('DEPENDS') or "")) |
| rcpinfo.packages = packages |
| rcpinfo.layer = layer |
| write_recipehistory(rcpinfo, d) |
| |
| pkgdest = d.getVar('PKGDEST') |
| for pkg in packagelist: |
| pkgdata = {} |
| with open(os.path.join(pkgdata_dir, 'runtime', pkg)) as f: |
| for line in f.readlines(): |
| item = line.rstrip('\n').split(': ', 1) |
| key = item[0] |
| if key.endswith('_' + pkg): |
| key = key[:-len(pkg)-1] |
| pkgdata[key] = item[1] |
| |
| pkge = pkgdata.get('PKGE', '0') |
| pkgv = pkgdata['PKGV'] |
| pkgr = pkgdata['PKGR'] |
| # |
| # Find out what the last version was |
| # Make sure the version did not decrease |
| # |
| lastversion = getlastpkgversion(pkg) |
| if lastversion: |
| last_pkge = lastversion.pkge |
| last_pkgv = lastversion.pkgv |
| last_pkgr = lastversion.pkgr |
| r = bb.utils.vercmp((pkge, pkgv, pkgr), (last_pkge, last_pkgv, last_pkgr)) |
| if r < 0: |
| msg = "Package version for package %s went backwards which would break package feeds from (%s:%s-%s to %s:%s-%s)" % (pkg, last_pkge, last_pkgv, last_pkgr, pkge, pkgv, pkgr) |
| package_qa_handle_error("version-going-backwards", msg, d) |
| |
| pkginfo = PackageInfo(pkg) |
| # Apparently the version can be different on a per-package basis (see Python) |
| pkginfo.pe = pkgdata.get('PE', '0') |
| pkginfo.pv = pkgdata['PV'] |
| pkginfo.pr = pkgdata['PR'] |
| pkginfo.pkg = pkgdata['PKG'] |
| pkginfo.pkge = pkge |
| pkginfo.pkgv = pkgv |
| pkginfo.pkgr = pkgr |
| pkginfo.rprovides = sortpkglist(oe.utils.squashspaces(pkgdata.get('RPROVIDES', ""))) |
| pkginfo.rdepends = sortpkglist(oe.utils.squashspaces(pkgdata.get('RDEPENDS', ""))) |
| pkginfo.rrecommends = sortpkglist(oe.utils.squashspaces(pkgdata.get('RRECOMMENDS', ""))) |
| pkginfo.rsuggests = sortpkglist(oe.utils.squashspaces(pkgdata.get('RSUGGESTS', ""))) |
| pkginfo.rreplaces = sortpkglist(oe.utils.squashspaces(pkgdata.get('RREPLACES', ""))) |
| pkginfo.rconflicts = sortpkglist(oe.utils.squashspaces(pkgdata.get('RCONFLICTS', ""))) |
| pkginfo.files = oe.utils.squashspaces(pkgdata.get('FILES', "")) |
| for filevar in pkginfo.filevars: |
| pkginfo.filevars[filevar] = pkgdata.get(filevar, "") |
| |
| # Gather information about packaged files |
| val = pkgdata.get('FILES_INFO', '') |
| dictval = json.loads(val) |
| filelist = list(dictval.keys()) |
| filelist.sort() |
| pkginfo.filelist = " ".join(filelist) |
| |
| pkginfo.size = int(pkgdata['PKGSIZE']) |
| |
| write_pkghistory(pkginfo, d) |
| |
| # Create files-in-<package-name>.txt files containing a list of files of each recipe's package |
| bb.build.exec_func("buildhistory_list_pkg_files", d) |
| } |
| |
| python buildhistory_emit_outputsigs() { |
| if not "task" in (d.getVar('BUILDHISTORY_FEATURES') or "").split(): |
| return |
| |
| import hashlib |
| |
| taskoutdir = os.path.join(d.getVar('BUILDHISTORY_DIR'), 'task', 'output') |
| bb.utils.mkdirhier(taskoutdir) |
| currenttask = d.getVar('BB_CURRENTTASK') |
| pn = d.getVar('PN') |
| taskfile = os.path.join(taskoutdir, '%s.%s' % (pn, currenttask)) |
| |
| cwd = os.getcwd() |
| filesigs = {} |
| for root, _, files in os.walk(cwd): |
| for fname in files: |
| if fname == 'fixmepath': |
| continue |
| fullpath = os.path.join(root, fname) |
| try: |
| if os.path.islink(fullpath): |
| sha256 = hashlib.sha256(os.readlink(fullpath).encode('utf-8')).hexdigest() |
| elif os.path.isfile(fullpath): |
| sha256 = bb.utils.sha256_file(fullpath) |
| else: |
| continue |
| except OSError: |
| bb.warn('buildhistory: unable to read %s to get output signature' % fullpath) |
| continue |
| filesigs[os.path.relpath(fullpath, cwd)] = sha256 |
| with open(taskfile, 'w') as f: |
| for fpath, fsig in sorted(filesigs.items(), key=lambda item: item[0]): |
| f.write('%s %s\n' % (fpath, fsig)) |
| } |
| |
| |
| def write_recipehistory(rcpinfo, d): |
| bb.debug(2, "Writing recipe history") |
| |
| pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE') |
| |
| infofile = os.path.join(pkghistdir, "latest") |
| with open(infofile, "w") as f: |
| if rcpinfo.pe != "0": |
| f.write(u"PE = %s\n" % rcpinfo.pe) |
| f.write(u"PV = %s\n" % rcpinfo.pv) |
| f.write(u"PR = %s\n" % rcpinfo.pr) |
| f.write(u"DEPENDS = %s\n" % rcpinfo.depends) |
| f.write(u"PACKAGES = %s\n" % rcpinfo.packages) |
| f.write(u"LAYER = %s\n" % rcpinfo.layer) |
| |
| write_latest_srcrev(d, pkghistdir) |
| |
| def write_pkghistory(pkginfo, d): |
| bb.debug(2, "Writing package history for package %s" % pkginfo.name) |
| |
| pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE') |
| |
| pkgpath = os.path.join(pkghistdir, pkginfo.name) |
| if not os.path.exists(pkgpath): |
| bb.utils.mkdirhier(pkgpath) |
| |
| infofile = os.path.join(pkgpath, "latest") |
| with open(infofile, "w") as f: |
| if pkginfo.pe != "0": |
| f.write(u"PE = %s\n" % pkginfo.pe) |
| f.write(u"PV = %s\n" % pkginfo.pv) |
| f.write(u"PR = %s\n" % pkginfo.pr) |
| |
| if pkginfo.pkg != pkginfo.name: |
| f.write(u"PKG = %s\n" % pkginfo.pkg) |
| if pkginfo.pkge != pkginfo.pe: |
| f.write(u"PKGE = %s\n" % pkginfo.pkge) |
| if pkginfo.pkgv != pkginfo.pv: |
| f.write(u"PKGV = %s\n" % pkginfo.pkgv) |
| if pkginfo.pkgr != pkginfo.pr: |
| f.write(u"PKGR = %s\n" % pkginfo.pkgr) |
| f.write(u"RPROVIDES = %s\n" % pkginfo.rprovides) |
| f.write(u"RDEPENDS = %s\n" % pkginfo.rdepends) |
| f.write(u"RRECOMMENDS = %s\n" % pkginfo.rrecommends) |
| if pkginfo.rsuggests: |
| f.write(u"RSUGGESTS = %s\n" % pkginfo.rsuggests) |
| if pkginfo.rreplaces: |
| f.write(u"RREPLACES = %s\n" % pkginfo.rreplaces) |
| if pkginfo.rconflicts: |
| f.write(u"RCONFLICTS = %s\n" % pkginfo.rconflicts) |
| f.write(u"PKGSIZE = %d\n" % pkginfo.size) |
| f.write(u"FILES = %s\n" % pkginfo.files) |
| f.write(u"FILELIST = %s\n" % pkginfo.filelist) |
| |
| for filevar in pkginfo.filevars: |
| filevarpath = os.path.join(pkgpath, "latest.%s" % filevar) |
| val = pkginfo.filevars[filevar] |
| if val: |
| with open(filevarpath, "w") as f: |
| f.write(val) |
| else: |
| if os.path.exists(filevarpath): |
| os.unlink(filevarpath) |
| |
| # |
| # rootfs_type can be: image, sdk_target, sdk_host |
| # |
| def buildhistory_list_installed(d, rootfs_type="image"): |
| from oe.rootfs import image_list_installed_packages |
| from oe.sdk import sdk_list_installed_packages |
| from oe.utils import format_pkg_list |
| |
| process_list = [('file', 'bh_installed_pkgs.txt'),\ |
| ('deps', 'bh_installed_pkgs_deps.txt')] |
| |
| if rootfs_type == "image": |
| pkgs = image_list_installed_packages(d) |
| else: |
| pkgs = sdk_list_installed_packages(d, rootfs_type == "sdk_target") |
| |
| for output_type, output_file in process_list: |
| output_file_full = os.path.join(d.getVar('WORKDIR'), output_file) |
| |
| with open(output_file_full, 'w') as output: |
| output.write(format_pkg_list(pkgs, output_type)) |
| |
| python buildhistory_list_installed_image() { |
| buildhistory_list_installed(d) |
| } |
| |
| python buildhistory_list_installed_sdk_target() { |
| buildhistory_list_installed(d, "sdk_target") |
| } |
| |
| python buildhistory_list_installed_sdk_host() { |
| buildhistory_list_installed(d, "sdk_host") |
| } |
| |
| buildhistory_get_installed() { |
| mkdir -p $1 |
| |
| # Get list of installed packages |
| pkgcache="$1/installed-packages.tmp" |
| cat ${WORKDIR}/bh_installed_pkgs.txt | sort > $pkgcache && rm ${WORKDIR}/bh_installed_pkgs.txt |
| |
| cat $pkgcache | awk '{ print $1 }' > $1/installed-package-names.txt |
| if [ -s $pkgcache ] ; then |
| cat $pkgcache | awk '{ print $2 }' | xargs -n1 basename > $1/installed-packages.txt |
| else |
| printf "" > $1/installed-packages.txt |
| fi |
| |
| # Produce dependency graph |
| # First, quote each name to handle characters that cause issues for dot |
| sed 's:\([^| ]*\):"\1":g' ${WORKDIR}/bh_installed_pkgs_deps.txt > $1/depends.tmp && |
| rm ${WORKDIR}/bh_installed_pkgs_deps.txt |
| # Remove lines with rpmlib(...) and config(...) dependencies, change the |
| # delimiter from pipe to "->", set the style for recommend lines and |
| # turn versioned dependencies into edge labels. |
| sed -i -e '/rpmlib(/d' \ |
| -e '/config(/d' \ |
| -e 's:|: -> :' \ |
| -e 's:"\[REC\]":[style=dotted]:' \ |
| -e 's:"\([<>=]\+\)" "\([^"]*\)":[label="\1 \2"]:' \ |
| $1/depends.tmp |
| # Add header, sorted and de-duped contents and footer and then delete the temp file |
| printf "digraph depends {\n node [shape=plaintext]\n" > $1/depends.dot |
| cat $1/depends.tmp | sort -u >> $1/depends.dot |
| echo "}" >> $1/depends.dot |
| rm $1/depends.tmp |
| |
| # Produce installed package sizes list |
| oe-pkgdata-util -p ${PKGDATA_DIR} read-value "PKGSIZE" -n -f $pkgcache > $1/installed-package-sizes.tmp |
| cat $1/installed-package-sizes.tmp | awk '{print $2 "\tKiB\t" $1}' | sort -n -r > $1/installed-package-sizes.txt |
| rm $1/installed-package-sizes.tmp |
| |
| # We're now done with the cache, delete it |
| rm $pkgcache |
| |
| if [ "$2" != "sdk" ] ; then |
| # Produce some cut-down graphs (for readability) |
| grep -v kernel-image $1/depends.dot | grep -v kernel-3 | grep -v kernel-4 > $1/depends-nokernel.dot |
| grep -v libc6 $1/depends-nokernel.dot | grep -v libgcc > $1/depends-nokernel-nolibc.dot |
| grep -v update- $1/depends-nokernel-nolibc.dot > $1/depends-nokernel-nolibc-noupdate.dot |
| grep -v kernel-module $1/depends-nokernel-nolibc-noupdate.dot > $1/depends-nokernel-nolibc-noupdate-nomodules.dot |
| fi |
| |
| # add complementary package information |
| if [ -e ${WORKDIR}/complementary_pkgs.txt ]; then |
| cp ${WORKDIR}/complementary_pkgs.txt $1 |
| fi |
| } |
| |
| buildhistory_get_image_installed() { |
| # Anything requiring the use of the packaging system should be done in here |
| # in case the packaging files are going to be removed for this image |
| |
| if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'image', '1', '0', d)}" = "0" ] ; then |
| return |
| fi |
| |
| buildhistory_get_installed ${BUILDHISTORY_DIR_IMAGE} |
| } |
| |
| buildhistory_get_sdk_installed() { |
| # Anything requiring the use of the packaging system should be done in here |
| # in case the packaging files are going to be removed for this SDK |
| |
| if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'sdk', '1', '0', d)}" = "0" ] ; then |
| return |
| fi |
| |
| buildhistory_get_installed ${BUILDHISTORY_DIR_SDK}/$1 sdk |
| } |
| |
| buildhistory_get_sdk_installed_host() { |
| buildhistory_get_sdk_installed host |
| } |
| |
| buildhistory_get_sdk_installed_target() { |
| buildhistory_get_sdk_installed target |
| } |
| |
| buildhistory_list_files() { |
| # List the files in the specified directory, but exclude date/time etc. |
| # This awk script is somewhat messy, but handles where the size is not printed for device files under pseudo |
| if [ "$3" = "fakeroot" ] ; then |
| ( cd $1 && ${FAKEROOTENV} ${FAKEROOTCMD} find . ! -path . -printf "%M %-10u %-10g %10s %p -> %l\n" | sort -k5 | sed 's/ * -> $//' > $2 ) |
| else |
| ( cd $1 && find . ! -path . -printf "%M %-10u %-10g %10s %p -> %l\n" | sort -k5 | sed 's/ * -> $//' > $2 ) |
| fi |
| } |
| |
| buildhistory_list_pkg_files() { |
| # Create individual files-in-package for each recipe's package |
| for pkgdir in $(find ${PKGDEST}/* -maxdepth 0 -type d); do |
| pkgname=$(basename $pkgdir) |
| outfolder="${BUILDHISTORY_DIR_PACKAGE}/$pkgname" |
| outfile="$outfolder/files-in-package.txt" |
| # Make sure the output folder exists so we can create the file |
| if [ ! -d $outfolder ] ; then |
| bbdebug 2 "Folder $outfolder does not exist, file $outfile not created" |
| continue |
| fi |
| buildhistory_list_files $pkgdir $outfile fakeroot |
| done |
| } |
| |
| buildhistory_get_imageinfo() { |
| if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'image', '1', '0', d)}" = "0" ] ; then |
| return |
| fi |
| |
| mkdir -p ${BUILDHISTORY_DIR_IMAGE} |
| buildhistory_list_files ${IMAGE_ROOTFS} ${BUILDHISTORY_DIR_IMAGE}/files-in-image.txt |
| |
| # Collect files requested in BUILDHISTORY_IMAGE_FILES |
| rm -rf ${BUILDHISTORY_DIR_IMAGE}/image-files |
| for f in ${BUILDHISTORY_IMAGE_FILES}; do |
| if [ -f ${IMAGE_ROOTFS}/$f ] ; then |
| mkdir -p ${BUILDHISTORY_DIR_IMAGE}/image-files/`dirname $f` |
| cp ${IMAGE_ROOTFS}/$f ${BUILDHISTORY_DIR_IMAGE}/image-files/$f |
| fi |
| done |
| |
| # Record some machine-readable meta-information about the image |
| printf "" > ${BUILDHISTORY_DIR_IMAGE}/image-info.txt |
| cat >> ${BUILDHISTORY_DIR_IMAGE}/image-info.txt <<END |
| ${@buildhistory_get_imagevars(d)} |
| END |
| imagesize=`du -ks ${IMAGE_ROOTFS} | awk '{ print $1 }'` |
| echo "IMAGESIZE = $imagesize" >> ${BUILDHISTORY_DIR_IMAGE}/image-info.txt |
| |
| # Add some configuration information |
| echo "${MACHINE}: ${IMAGE_BASENAME} configured for ${DISTRO} ${DISTRO_VERSION}" > ${BUILDHISTORY_DIR_IMAGE}/build-id.txt |
| |
| cat >> ${BUILDHISTORY_DIR_IMAGE}/build-id.txt <<END |
| ${@buildhistory_get_build_id(d)} |
| END |
| } |
| |
| buildhistory_get_sdkinfo() { |
| if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'sdk', '1', '0', d)}" = "0" ] ; then |
| return |
| fi |
| |
| buildhistory_list_files ${SDK_OUTPUT} ${BUILDHISTORY_DIR_SDK}/files-in-sdk.txt |
| |
| # Collect files requested in BUILDHISTORY_SDK_FILES |
| rm -rf ${BUILDHISTORY_DIR_SDK}/sdk-files |
| for f in ${BUILDHISTORY_SDK_FILES}; do |
| if [ -f ${SDK_OUTPUT}/${SDKPATH}/$f ] ; then |
| mkdir -p ${BUILDHISTORY_DIR_SDK}/sdk-files/`dirname $f` |
| cp ${SDK_OUTPUT}/${SDKPATH}/$f ${BUILDHISTORY_DIR_SDK}/sdk-files/$f |
| fi |
| done |
| |
| # Record some machine-readable meta-information about the SDK |
| printf "" > ${BUILDHISTORY_DIR_SDK}/sdk-info.txt |
| cat >> ${BUILDHISTORY_DIR_SDK}/sdk-info.txt <<END |
| ${@buildhistory_get_sdkvars(d)} |
| END |
| sdksize=`du -ks ${SDK_OUTPUT} | awk '{ print $1 }'` |
| echo "SDKSIZE = $sdksize" >> ${BUILDHISTORY_DIR_SDK}/sdk-info.txt |
| } |
| |
| python buildhistory_get_extra_sdkinfo() { |
| import operator |
| from oe.sdk import get_extra_sdkinfo |
| |
| sstate_dir = d.expand('${SDK_OUTPUT}/${SDKPATH}/sstate-cache') |
| extra_info = get_extra_sdkinfo(sstate_dir) |
| |
| if d.getVar('BB_CURRENTTASK') == 'populate_sdk_ext' and \ |
| "sdk" in (d.getVar('BUILDHISTORY_FEATURES') or "").split(): |
| with open(d.expand('${BUILDHISTORY_DIR_SDK}/sstate-package-sizes.txt'), 'w') as f: |
| filesizes_sorted = sorted(extra_info['filesizes'].items(), key=operator.itemgetter(1, 0), reverse=True) |
| for fn, size in filesizes_sorted: |
| f.write('%10d KiB %s\n' % (size, fn)) |
| with open(d.expand('${BUILDHISTORY_DIR_SDK}/sstate-task-sizes.txt'), 'w') as f: |
| tasksizes_sorted = sorted(extra_info['tasksizes'].items(), key=operator.itemgetter(1, 0), reverse=True) |
| for task, size in tasksizes_sorted: |
| f.write('%10d KiB %s\n' % (size, task)) |
| } |
| |
| # By using ROOTFS_POSTUNINSTALL_COMMAND we get in after uninstallation of |
| # unneeded packages but before the removal of packaging files |
| ROOTFS_POSTUNINSTALL_COMMAND += "buildhistory_list_installed_image ;" |
| ROOTFS_POSTUNINSTALL_COMMAND += "buildhistory_get_image_installed ;" |
| ROOTFS_POSTUNINSTALL_COMMAND[vardepvalueexclude] .= "| buildhistory_list_installed_image ;| buildhistory_get_image_installed ;" |
| ROOTFS_POSTUNINSTALL_COMMAND[vardepsexclude] += "buildhistory_list_installed_image buildhistory_get_image_installed" |
| |
| IMAGE_POSTPROCESS_COMMAND += "buildhistory_get_imageinfo ;" |
| IMAGE_POSTPROCESS_COMMAND[vardepvalueexclude] .= "| buildhistory_get_imageinfo ;" |
| IMAGE_POSTPROCESS_COMMAND[vardepsexclude] += "buildhistory_get_imageinfo" |
| |
| # We want these to be the last run so that we get called after complementary package installation |
| POPULATE_SDK_POST_TARGET_COMMAND_append = " buildhistory_list_installed_sdk_target;" |
| POPULATE_SDK_POST_TARGET_COMMAND_append = " buildhistory_get_sdk_installed_target;" |
| POPULATE_SDK_POST_TARGET_COMMAND[vardepvalueexclude] .= "| buildhistory_list_installed_sdk_target;| buildhistory_get_sdk_installed_target;" |
| |
| POPULATE_SDK_POST_HOST_COMMAND_append = " buildhistory_list_installed_sdk_host;" |
| POPULATE_SDK_POST_HOST_COMMAND_append = " buildhistory_get_sdk_installed_host;" |
| POPULATE_SDK_POST_HOST_COMMAND[vardepvalueexclude] .= "| buildhistory_list_installed_sdk_host;| buildhistory_get_sdk_installed_host;" |
| |
| SDK_POSTPROCESS_COMMAND_append = " buildhistory_get_sdkinfo ; buildhistory_get_extra_sdkinfo; " |
| SDK_POSTPROCESS_COMMAND[vardepvalueexclude] .= "| buildhistory_get_sdkinfo ; buildhistory_get_extra_sdkinfo; " |
| |
| python buildhistory_write_sigs() { |
| if not "task" in (d.getVar('BUILDHISTORY_FEATURES') or "").split(): |
| return |
| |
| # Create sigs file |
| if hasattr(bb.parse.siggen, 'dump_siglist'): |
| taskoutdir = os.path.join(d.getVar('BUILDHISTORY_DIR'), 'task') |
| bb.utils.mkdirhier(taskoutdir) |
| bb.parse.siggen.dump_siglist(os.path.join(taskoutdir, 'tasksigs.txt')) |
| } |
| |
| def buildhistory_get_build_id(d): |
| if d.getVar('BB_WORKERCONTEXT') != '1': |
| return "" |
| localdata = bb.data.createCopy(d) |
| statuslines = [] |
| for func in oe.data.typed_value('BUILDCFG_FUNCS', localdata): |
| g = globals() |
| if func not in g: |
| bb.warn("Build configuration function '%s' does not exist" % func) |
| else: |
| flines = g[func](localdata) |
| if flines: |
| statuslines.extend(flines) |
| |
| statusheader = d.getVar('BUILDCFG_HEADER') |
| return('\n%s\n%s\n' % (statusheader, '\n'.join(statuslines))) |
| |
| def buildhistory_get_metadata_revs(d): |
| # We want an easily machine-readable format here, so get_layers_branch_rev isn't quite what we want |
| layers = (d.getVar("BBLAYERS") or "").split() |
| medadata_revs = ["%-17s = %s:%s" % (os.path.basename(i), \ |
| base_get_metadata_git_branch(i, None).strip(), \ |
| base_get_metadata_git_revision(i, None)) \ |
| for i in layers] |
| return '\n'.join(medadata_revs) |
| |
| def outputvars(vars, listvars, d): |
| vars = vars.split() |
| listvars = listvars.split() |
| ret = "" |
| for var in vars: |
| value = d.getVar(var) or "" |
| if var in listvars: |
| # Squash out spaces |
| value = oe.utils.squashspaces(value) |
| ret += "%s = %s\n" % (var, value) |
| return ret.rstrip('\n') |
| |
| def buildhistory_get_imagevars(d): |
| if d.getVar('BB_WORKERCONTEXT') != '1': |
| return "" |
| imagevars = "DISTRO DISTRO_VERSION USER_CLASSES IMAGE_CLASSES IMAGE_FEATURES IMAGE_LINGUAS IMAGE_INSTALL BAD_RECOMMENDATIONS NO_RECOMMENDATIONS PACKAGE_EXCLUDE ROOTFS_POSTPROCESS_COMMAND IMAGE_POSTPROCESS_COMMAND" |
| listvars = "USER_CLASSES IMAGE_CLASSES IMAGE_FEATURES IMAGE_LINGUAS IMAGE_INSTALL BAD_RECOMMENDATIONS PACKAGE_EXCLUDE" |
| return outputvars(imagevars, listvars, d) |
| |
| def buildhistory_get_sdkvars(d): |
| if d.getVar('BB_WORKERCONTEXT') != '1': |
| return "" |
| sdkvars = "DISTRO DISTRO_VERSION SDK_NAME SDK_VERSION SDKMACHINE SDKIMAGE_FEATURES BAD_RECOMMENDATIONS NO_RECOMMENDATIONS PACKAGE_EXCLUDE" |
| if d.getVar('BB_CURRENTTASK') == 'populate_sdk_ext': |
| # Extensible SDK uses some additional variables |
| sdkvars += " SDK_LOCAL_CONF_WHITELIST SDK_LOCAL_CONF_BLACKLIST SDK_INHERIT_BLACKLIST SDK_UPDATE_URL SDK_EXT_TYPE SDK_RECRDEP_TASKS SDK_INCLUDE_PKGDATA SDK_INCLUDE_TOOLCHAIN" |
| listvars = "SDKIMAGE_FEATURES BAD_RECOMMENDATIONS PACKAGE_EXCLUDE SDK_LOCAL_CONF_WHITELIST SDK_LOCAL_CONF_BLACKLIST SDK_INHERIT_BLACKLIST" |
| return outputvars(sdkvars, listvars, d) |
| |
| |
| def buildhistory_get_cmdline(d): |
| argv = d.getVar('BB_CMDLINE', False) |
| if argv: |
| if argv[0].endswith('bin/bitbake'): |
| bincmd = 'bitbake' |
| else: |
| bincmd = argv[0] |
| return '%s %s' % (bincmd, ' '.join(argv[1:])) |
| return '' |
| |
| |
| buildhistory_single_commit() { |
| if [ "$3" = "" ] ; then |
| commitopts="${BUILDHISTORY_DIR}/ --allow-empty" |
| shortlogprefix="No changes: " |
| else |
| commitopts="" |
| shortlogprefix="" |
| fi |
| if [ "${BUILDHISTORY_BUILD_FAILURES}" = "0" ] ; then |
| result="succeeded" |
| else |
| result="failed" |
| fi |
| case ${BUILDHISTORY_BUILD_INTERRUPTED} in |
| 1) |
| result="$result (interrupted)" |
| ;; |
| 2) |
| result="$result (force interrupted)" |
| ;; |
| esac |
| commitmsgfile=`mktemp` |
| cat > $commitmsgfile << END |
| ${shortlogprefix}Build ${BUILDNAME} of ${DISTRO} ${DISTRO_VERSION} for machine ${MACHINE} on $2 |
| |
| cmd: $1 |
| |
| result: $result |
| |
| metadata revisions: |
| END |
| cat ${BUILDHISTORY_DIR}/metadata-revs >> $commitmsgfile |
| git commit $commitopts -F $commitmsgfile --author "${BUILDHISTORY_COMMIT_AUTHOR}" > /dev/null |
| rm $commitmsgfile |
| } |
| |
| buildhistory_commit() { |
| if [ ! -d ${BUILDHISTORY_DIR} ] ; then |
| # Code above that creates this dir never executed, so there can't be anything to commit |
| return |
| fi |
| |
| # Create a machine-readable list of metadata revisions for each layer |
| cat > ${BUILDHISTORY_DIR}/metadata-revs <<END |
| ${@buildhistory_get_metadata_revs(d)} |
| END |
| |
| ( cd ${BUILDHISTORY_DIR}/ |
| # Initialise the repo if necessary |
| if [ ! -e .git ] ; then |
| git init -q |
| else |
| git tag -f build-minus-3 build-minus-2 > /dev/null 2>&1 || true |
| git tag -f build-minus-2 build-minus-1 > /dev/null 2>&1 || true |
| git tag -f build-minus-1 > /dev/null 2>&1 || true |
| fi |
| |
| check_git_config |
| |
| # Check if there are new/changed files to commit (other than metadata-revs) |
| repostatus=`git status --porcelain | grep -v " metadata-revs$"` |
| HOSTNAME=`hostname 2>/dev/null || echo unknown` |
| CMDLINE="${@buildhistory_get_cmdline(d)}" |
| if [ "$repostatus" != "" ] ; then |
| git add -A . |
| # porcelain output looks like "?? packages/foo/bar" |
| # Ensure we commit metadata-revs with the first commit |
| buildhistory_single_commit "$CMDLINE" "$HOSTNAME" dummy |
| git gc --auto --quiet |
| else |
| buildhistory_single_commit "$CMDLINE" "$HOSTNAME" |
| fi |
| if [ "${BUILDHISTORY_PUSH_REPO}" != "" ] ; then |
| git push -q ${BUILDHISTORY_PUSH_REPO} |
| fi) || true |
| } |
| |
| python buildhistory_eventhandler() { |
| if e.data.getVar('BUILDHISTORY_FEATURES').strip(): |
| reset = e.data.getVar("BUILDHISTORY_RESET") |
| olddir = e.data.getVar("BUILDHISTORY_OLD_DIR") |
| if isinstance(e, bb.event.BuildStarted): |
| if reset: |
| import shutil |
| # Clean up after potentially interrupted build. |
| if os.path.isdir(olddir): |
| shutil.rmtree(olddir) |
| rootdir = e.data.getVar("BUILDHISTORY_DIR") |
| entries = [ x for x in os.listdir(rootdir) if not x.startswith('.') ] |
| bb.utils.mkdirhier(olddir) |
| for entry in entries: |
| os.rename(os.path.join(rootdir, entry), |
| os.path.join(olddir, entry)) |
| elif isinstance(e, bb.event.BuildCompleted): |
| if reset: |
| import shutil |
| shutil.rmtree(olddir) |
| if e.data.getVar("BUILDHISTORY_COMMIT") == "1": |
| bb.note("Writing buildhistory") |
| bb.build.exec_func("buildhistory_write_sigs", d) |
| localdata = bb.data.createCopy(e.data) |
| localdata.setVar('BUILDHISTORY_BUILD_FAILURES', str(e._failures)) |
| interrupted = getattr(e, '_interrupted', 0) |
| localdata.setVar('BUILDHISTORY_BUILD_INTERRUPTED', str(interrupted)) |
| bb.build.exec_func("buildhistory_commit", localdata) |
| else: |
| bb.note("No commit since BUILDHISTORY_COMMIT != '1'") |
| } |
| |
| addhandler buildhistory_eventhandler |
| buildhistory_eventhandler[eventmask] = "bb.event.BuildCompleted bb.event.BuildStarted" |
| |
| |
| # FIXME this ought to be moved into the fetcher |
| def _get_srcrev_values(d): |
| """ |
| Return the version strings for the current recipe |
| """ |
| |
| scms = [] |
| fetcher = bb.fetch.Fetch(d.getVar('SRC_URI').split(), d) |
| urldata = fetcher.ud |
| for u in urldata: |
| if urldata[u].method.supports_srcrev(): |
| scms.append(u) |
| |
| autoinc_templ = 'AUTOINC+' |
| dict_srcrevs = {} |
| dict_tag_srcrevs = {} |
| for scm in scms: |
| ud = urldata[scm] |
| for name in ud.names: |
| try: |
| rev = ud.method.sortable_revision(ud, d, name) |
| except TypeError: |
| # support old bitbake versions |
| rev = ud.method.sortable_revision(scm, ud, d, name) |
| # Clean this up when we next bump bitbake version |
| if type(rev) != str: |
| autoinc, rev = rev |
| elif rev.startswith(autoinc_templ): |
| rev = rev[len(autoinc_templ):] |
| dict_srcrevs[name] = rev |
| if 'tag' in ud.parm: |
| tag = ud.parm['tag']; |
| key = name+'_'+tag |
| dict_tag_srcrevs[key] = rev |
| return (dict_srcrevs, dict_tag_srcrevs) |
| |
| do_fetch[postfuncs] += "write_srcrev" |
| do_fetch[vardepsexclude] += "write_srcrev" |
| python write_srcrev() { |
| write_latest_srcrev(d, d.getVar('BUILDHISTORY_DIR_PACKAGE')) |
| } |
| |
| def write_latest_srcrev(d, pkghistdir): |
| srcrevfile = os.path.join(pkghistdir, 'latest_srcrev') |
| |
| srcrevs, tag_srcrevs = _get_srcrev_values(d) |
| if srcrevs: |
| if not os.path.exists(pkghistdir): |
| bb.utils.mkdirhier(pkghistdir) |
| old_tag_srcrevs = {} |
| if os.path.exists(srcrevfile): |
| with open(srcrevfile) as f: |
| for line in f: |
| if line.startswith('# tag_'): |
| key, value = line.split("=", 1) |
| key = key.replace('# tag_', '').strip() |
| value = value.replace('"', '').strip() |
| old_tag_srcrevs[key] = value |
| with open(srcrevfile, 'w') as f: |
| orig_srcrev = d.getVar('SRCREV', False) or 'INVALID' |
| if orig_srcrev != 'INVALID': |
| f.write('# SRCREV = "%s"\n' % orig_srcrev) |
| if len(srcrevs) > 1: |
| for name, srcrev in srcrevs.items(): |
| orig_srcrev = d.getVar('SRCREV_%s' % name, False) |
| if orig_srcrev: |
| f.write('# SRCREV_%s = "%s"\n' % (name, orig_srcrev)) |
| f.write('SRCREV_%s = "%s"\n' % (name, srcrev)) |
| else: |
| f.write('SRCREV = "%s"\n' % next(iter(srcrevs.values()))) |
| if len(tag_srcrevs) > 0: |
| for name, srcrev in tag_srcrevs.items(): |
| f.write('# tag_%s = "%s"\n' % (name, srcrev)) |
| if name in old_tag_srcrevs and old_tag_srcrevs[name] != srcrev: |
| pkg = d.getVar('PN') |
| bb.warn("Revision for tag %s in package %s was changed since last build (from %s to %s)" % (name, pkg, old_tag_srcrevs[name], srcrev)) |
| |
| else: |
| if os.path.exists(srcrevfile): |
| os.remove(srcrevfile) |
| |
| do_testimage[postfuncs] += "write_ptest_result" |
| do_testimage[vardepsexclude] += "write_ptest_result" |
| |
| python write_ptest_result() { |
| write_latest_ptest_result(d, d.getVar('BUILDHISTORY_DIR')) |
| } |
| |
| def write_latest_ptest_result(d, histdir): |
| import glob |
| import subprocess |
| test_log_dir = d.getVar('TEST_LOG_DIR') |
| input_ptest = os.path.join(test_log_dir, 'ptest_log') |
| output_ptest = os.path.join(histdir, 'ptest') |
| if os.path.exists(input_ptest): |
| try: |
| # Lock it avoid race issue |
| lock = bb.utils.lockfile(output_ptest + "/ptest.lock") |
| bb.utils.mkdirhier(output_ptest) |
| oe.path.copytree(input_ptest, output_ptest) |
| # Sort test result |
| for result in glob.glob('%s/pass.fail.*' % output_ptest): |
| bb.debug(1, 'Processing %s' % result) |
| cmd = ['sort', result, '-o', result] |
| bb.debug(1, 'Running %s' % cmd) |
| ret = subprocess.call(cmd) |
| if ret != 0: |
| bb.error('Failed to run %s!' % cmd) |
| finally: |
| bb.utils.unlockfile(lock) |