| # Class to avoid copying packages into the feed if they haven't materially changed |
| # |
| # Copyright (C) 2015 Intel Corporation |
| # Released under the MIT license (see COPYING.MIT for details) |
| # |
| # This class effectively intercepts packages as they are written out by |
| # do_package_write_*, causing them to be written into a different |
| # directory where we can compare them to whatever older packages might |
| # be in the "real" package feed directory, and avoid copying the new |
| # package to the feed if it has not materially changed. The idea is to |
| # avoid unnecessary churn in the packages when dependencies trigger task |
| # reexecution (and thus repackaging). Enabling the class is simple: |
| # |
| # INHERIT += "packagefeed-stability" |
| # |
| # Caveats: |
| # 1) Latest PR values in the build system may not match those in packages |
| # seen on the target (naturally) |
| # 2) If you rebuild from sstate without the existing package feed present, |
| # you will lose the "state" of the package feed i.e. the preserved old |
| # package versions. Not the end of the world, but would negate the |
| # entire purpose of this class. |
| # |
| # Note that running -c cleanall on a recipe will purposely delete the old |
| # package files so they will definitely be copied the next time. |
| |
| python() { |
| if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross', d): |
| return |
| # Package backend agnostic intercept |
| # This assumes that the package_write task is called package_write_<pkgtype> |
| # and that the directory in which packages should be written is |
| # pointed to by the variable DEPLOY_DIR_<PKGTYPE> |
| for pkgclass in (d.getVar('PACKAGE_CLASSES') or '').split(): |
| if pkgclass.startswith('package_'): |
| pkgtype = pkgclass.split('_', 1)[1] |
| pkgwritefunc = 'do_package_write_%s' % pkgtype |
| sstate_outputdirs = d.getVarFlag(pkgwritefunc, 'sstate-outputdirs', False) |
| deploydirvar = 'DEPLOY_DIR_%s' % pkgtype.upper() |
| deploydirvarref = '${' + deploydirvar + '}' |
| pkgcomparefunc = 'do_package_compare_%s' % pkgtype |
| |
| if bb.data.inherits_class('image', d): |
| d.appendVarFlag('do_rootfs', 'recrdeptask', ' ' + pkgcomparefunc) |
| |
| if bb.data.inherits_class('populate_sdk_base', d): |
| d.appendVarFlag('do_populate_sdk', 'recrdeptask', ' ' + pkgcomparefunc) |
| |
| if bb.data.inherits_class('populate_sdk_ext', d): |
| d.appendVarFlag('do_populate_sdk_ext', 'recrdeptask', ' ' + pkgcomparefunc) |
| |
| d.appendVarFlag('do_build', 'recrdeptask', ' ' + pkgcomparefunc) |
| |
| if d.getVarFlag(pkgwritefunc, 'noexec') or not d.getVarFlag(pkgwritefunc, 'task'): |
| # Packaging is disabled for this recipe, we shouldn't do anything |
| continue |
| |
| if deploydirvarref in sstate_outputdirs: |
| deplor_dir_pkgtype = d.expand(deploydirvarref + '-prediff') |
| # Set intermediate output directory |
| d.setVarFlag(pkgwritefunc, 'sstate-outputdirs', sstate_outputdirs.replace(deploydirvarref, deplor_dir_pkgtype)) |
| # Update SSTATE_DUPWHITELIST to avoid shared location conflicted error |
| d.appendVar('SSTATE_DUPWHITELIST', ' %s' % deplor_dir_pkgtype) |
| |
| d.setVar(pkgcomparefunc, d.getVar('do_package_compare', False)) |
| d.setVarFlags(pkgcomparefunc, d.getVarFlags('do_package_compare', False)) |
| d.appendVarFlag(pkgcomparefunc, 'depends', ' build-compare-native:do_populate_sysroot') |
| bb.build.addtask(pkgcomparefunc, 'do_build', 'do_packagedata ' + pkgwritefunc, d) |
| } |
| |
| # This isn't the real task function - it's a template that we use in the |
| # anonymous python code above |
| fakeroot python do_package_compare () { |
| currenttask = d.getVar('BB_CURRENTTASK') |
| pkgtype = currenttask.rsplit('_', 1)[1] |
| package_compare_impl(pkgtype, d) |
| } |
| |
| def package_compare_impl(pkgtype, d): |
| import errno |
| import fnmatch |
| import glob |
| import subprocess |
| import oe.sstatesig |
| |
| pn = d.getVar('PN') |
| deploydir = d.getVar('DEPLOY_DIR_%s' % pkgtype.upper()) |
| prepath = deploydir + '-prediff/' |
| |
| # Find out PKGR values are |
| pkgdatadir = d.getVar('PKGDATA_DIR') |
| packages = [] |
| try: |
| with open(os.path.join(pkgdatadir, pn), 'r') as f: |
| for line in f: |
| if line.startswith('PACKAGES:'): |
| packages = line.split(':', 1)[1].split() |
| break |
| except IOError as e: |
| if e.errno == errno.ENOENT: |
| pass |
| |
| if not packages: |
| bb.debug(2, '%s: no packages, nothing to do' % pn) |
| return |
| |
| pkgrvalues = {} |
| rpkgnames = {} |
| rdepends = {} |
| pkgvvalues = {} |
| for pkg in packages: |
| with open(os.path.join(pkgdatadir, 'runtime', pkg), 'r') as f: |
| for line in f: |
| if line.startswith('PKGR:'): |
| pkgrvalues[pkg] = line.split(':', 1)[1].strip() |
| if line.startswith('PKGV:'): |
| pkgvvalues[pkg] = line.split(':', 1)[1].strip() |
| elif line.startswith('PKG_%s:' % pkg): |
| rpkgnames[pkg] = line.split(':', 1)[1].strip() |
| elif line.startswith('RDEPENDS_%s:' % pkg): |
| rdepends[pkg] = line.split(':', 1)[1].strip() |
| |
| # Prepare a list of the runtime package names for packages that were |
| # actually produced |
| rpkglist = [] |
| for pkg, rpkg in rpkgnames.items(): |
| if os.path.exists(os.path.join(pkgdatadir, 'runtime', pkg + '.packaged')): |
| rpkglist.append((rpkg, pkg)) |
| rpkglist.sort(key=lambda x: len(x[0]), reverse=True) |
| |
| pvu = d.getVar('PV', False) |
| if '$' + '{SRCPV}' in pvu: |
| pvprefix = pvu.split('$' + '{SRCPV}', 1)[0] |
| else: |
| pvprefix = None |
| |
| pkgwritetask = 'package_write_%s' % pkgtype |
| files = [] |
| docopy = False |
| manifest, _ = oe.sstatesig.sstate_get_manifest_filename(pkgwritetask, d) |
| mlprefix = d.getVar('MLPREFIX') |
| # Copy recipe's all packages if one of the packages are different to make |
| # they have the same PR. |
| with open(manifest, 'r') as f: |
| for line in f: |
| if line.startswith(prepath): |
| srcpath = line.rstrip() |
| if os.path.isfile(srcpath): |
| destpath = os.path.join(deploydir, os.path.relpath(srcpath, prepath)) |
| |
| # This is crude but should work assuming the output |
| # package file name starts with the package name |
| # and rpkglist is sorted by length (descending) |
| pkgbasename = os.path.basename(destpath) |
| pkgname = None |
| for rpkg, pkg in rpkglist: |
| if mlprefix and pkgtype == 'rpm' and rpkg.startswith(mlprefix): |
| rpkg = rpkg[len(mlprefix):] |
| if pkgbasename.startswith(rpkg): |
| pkgr = pkgrvalues[pkg] |
| destpathspec = destpath.replace(pkgr, '*') |
| if pvprefix: |
| pkgv = pkgvvalues[pkg] |
| if pkgv.startswith(pvprefix): |
| pkgvsuffix = pkgv[len(pvprefix):] |
| if '+' in pkgvsuffix: |
| newpkgv = pvprefix + '*+' + pkgvsuffix.split('+', 1)[1] |
| destpathspec = destpathspec.replace(pkgv, newpkgv) |
| pkgname = pkg |
| break |
| else: |
| bb.warn('Unable to map %s back to package' % pkgbasename) |
| destpathspec = destpath |
| |
| oldfile = None |
| if not docopy: |
| oldfiles = glob.glob(destpathspec) |
| if oldfiles: |
| oldfile = oldfiles[-1] |
| result = subprocess.call(['pkg-diff.sh', oldfile, srcpath]) |
| if result != 0: |
| docopy = True |
| bb.note("%s and %s are different, will copy packages" % (oldfile, srcpath)) |
| else: |
| docopy = True |
| bb.note("No old packages found for %s, will copy packages" % pkgname) |
| |
| files.append((pkgname, pkgbasename, srcpath, destpath)) |
| |
| # Remove all the old files and copy again if docopy |
| if docopy: |
| bb.note('Copying packages for recipe %s' % pn) |
| pcmanifest = os.path.join(prepath, d.expand('pkg-compare-manifest-${MULTIMACH_TARGET_SYS}-${PN}')) |
| try: |
| with open(pcmanifest, 'r') as f: |
| for line in f: |
| fn = line.rstrip() |
| if fn: |
| try: |
| os.remove(fn) |
| bb.note('Removed old package %s' % fn) |
| except OSError as e: |
| if e.errno == errno.ENOENT: |
| pass |
| except IOError as e: |
| if e.errno == errno.ENOENT: |
| pass |
| |
| # Create new manifest |
| with open(pcmanifest, 'w') as f: |
| for pkgname, pkgbasename, srcpath, destpath in files: |
| destdir = os.path.dirname(destpath) |
| bb.utils.mkdirhier(destdir) |
| # Remove allarch rpm pkg if it is already existed (for |
| # multilib), they're identical in theory, but sstate.bbclass |
| # copies it again, so keep align with that. |
| if os.path.exists(destpath) and pkgtype == 'rpm' \ |
| and d.getVar('PACKAGE_ARCH') == 'all': |
| os.unlink(destpath) |
| if (os.stat(srcpath).st_dev == os.stat(destdir).st_dev): |
| # Use a hard link to save space |
| os.link(srcpath, destpath) |
| else: |
| shutil.copyfile(srcpath, destpath) |
| f.write('%s\n' % destpath) |
| else: |
| bb.note('Not copying packages for recipe %s' % pn) |
| |
| do_cleansstate[postfuncs] += "pfs_cleanpkgs" |
| python pfs_cleanpkgs () { |
| import errno |
| for pkgclass in (d.getVar('PACKAGE_CLASSES') or '').split(): |
| if pkgclass.startswith('package_'): |
| pkgtype = pkgclass.split('_', 1)[1] |
| deploydir = d.getVar('DEPLOY_DIR_%s' % pkgtype.upper()) |
| prepath = deploydir + '-prediff' |
| pcmanifest = os.path.join(prepath, d.expand('pkg-compare-manifest-${MULTIMACH_TARGET_SYS}-${PN}')) |
| try: |
| with open(pcmanifest, 'r') as f: |
| for line in f: |
| fn = line.rstrip() |
| if fn: |
| try: |
| os.remove(fn) |
| except OSError as e: |
| if e.errno == errno.ENOENT: |
| pass |
| os.remove(pcmanifest) |
| except IOError as e: |
| if e.errno == errno.ENOENT: |
| pass |
| } |