blob: aa01def74d1b0129ad1472a167ca2f761eaa836d [file] [log] [blame]
# Class to avoid copying packages into the feed if they haven't materially changed
#
# Copyright (C) 2015 Intel Corporation
# Released under the MIT license (see COPYING.MIT for details)
#
# This class effectively intercepts packages as they are written out by
# do_package_write_*, causing them to be written into a different
# directory where we can compare them to whatever older packages might
# be in the "real" package feed directory, and avoid copying the new
# package to the feed if it has not materially changed. The idea is to
# avoid unnecessary churn in the packages when dependencies trigger task
# reexecution (and thus repackaging). Enabling the class is simple:
#
# INHERIT += "packagefeed-stability"
#
# Caveats:
# 1) Latest PR values in the build system may not match those in packages
# seen on the target (naturally)
# 2) If you rebuild from sstate without the existing package feed present,
# you will lose the "state" of the package feed i.e. the preserved old
# package versions. Not the end of the world, but would negate the
# entire purpose of this class.
#
# Note that running -c cleanall on a recipe will purposely delete the old
# package files so they will definitely be copied the next time.
python() {
if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross', d):
return
# Package backend agnostic intercept
# This assumes that the package_write task is called package_write_<pkgtype>
# and that the directory in which packages should be written is
# pointed to by the variable DEPLOY_DIR_<PKGTYPE>
for pkgclass in (d.getVar('PACKAGE_CLASSES', True) or '').split():
if pkgclass.startswith('package_'):
pkgtype = pkgclass.split('_', 1)[1]
pkgwritefunc = 'do_package_write_%s' % pkgtype
sstate_outputdirs = d.getVarFlag(pkgwritefunc, 'sstate-outputdirs', False)
deploydirvar = 'DEPLOY_DIR_%s' % pkgtype.upper()
deploydirvarref = '${' + deploydirvar + '}'
pkgcomparefunc = 'do_package_compare_%s' % pkgtype
if bb.data.inherits_class('image', d):
d.appendVarFlag('do_rootfs', 'recrdeptask', ' ' + pkgcomparefunc)
if bb.data.inherits_class('populate_sdk_base', d):
d.appendVarFlag('do_populate_sdk', 'recrdeptask', ' ' + pkgcomparefunc)
if bb.data.inherits_class('populate_sdk_ext', d):
d.appendVarFlag('do_populate_sdk_ext', 'recrdeptask', ' ' + pkgcomparefunc)
d.appendVarFlag('do_build', 'recrdeptask', ' ' + pkgcomparefunc)
if d.getVarFlag(pkgwritefunc, 'noexec', True) or not d.getVarFlag(pkgwritefunc, 'task', True):
# Packaging is disabled for this recipe, we shouldn't do anything
continue
if deploydirvarref in sstate_outputdirs:
deplor_dir_pkgtype = d.expand(deploydirvarref + '-prediff')
# Set intermediate output directory
d.setVarFlag(pkgwritefunc, 'sstate-outputdirs', sstate_outputdirs.replace(deploydirvarref, deplor_dir_pkgtype))
# Update SSTATE_DUPWHITELIST to avoid shared location conflicted error
d.appendVar('SSTATE_DUPWHITELIST', ' %s' % deplor_dir_pkgtype)
d.setVar(pkgcomparefunc, d.getVar('do_package_compare', False))
d.setVarFlags(pkgcomparefunc, d.getVarFlags('do_package_compare', False))
d.appendVarFlag(pkgcomparefunc, 'depends', ' build-compare-native:do_populate_sysroot')
bb.build.addtask(pkgcomparefunc, 'do_build', 'do_packagedata ' + pkgwritefunc, d)
}
# This isn't the real task function - it's a template that we use in the
# anonymous python code above
fakeroot python do_package_compare () {
currenttask = d.getVar('BB_CURRENTTASK', True)
pkgtype = currenttask.rsplit('_', 1)[1]
package_compare_impl(pkgtype, d)
}
def package_compare_impl(pkgtype, d):
import errno
import fnmatch
import glob
import subprocess
import oe.sstatesig
pn = d.getVar('PN', True)
deploydir = d.getVar('DEPLOY_DIR_%s' % pkgtype.upper(), True)
prepath = deploydir + '-prediff/'
# Find out PKGR values are
pkgdatadir = d.getVar('PKGDATA_DIR', True)
packages = []
try:
with open(os.path.join(pkgdatadir, pn), 'r') as f:
for line in f:
if line.startswith('PACKAGES:'):
packages = line.split(':', 1)[1].split()
break
except IOError as e:
if e.errno == errno.ENOENT:
pass
if not packages:
bb.debug(2, '%s: no packages, nothing to do' % pn)
return
pkgrvalues = {}
rpkgnames = {}
rdepends = {}
pkgvvalues = {}
for pkg in packages:
with open(os.path.join(pkgdatadir, 'runtime', pkg), 'r') as f:
for line in f:
if line.startswith('PKGR:'):
pkgrvalues[pkg] = line.split(':', 1)[1].strip()
if line.startswith('PKGV:'):
pkgvvalues[pkg] = line.split(':', 1)[1].strip()
elif line.startswith('PKG_%s:' % pkg):
rpkgnames[pkg] = line.split(':', 1)[1].strip()
elif line.startswith('RDEPENDS_%s:' % pkg):
rdepends[pkg] = line.split(':', 1)[1].strip()
# Prepare a list of the runtime package names for packages that were
# actually produced
rpkglist = []
for pkg, rpkg in rpkgnames.items():
if os.path.exists(os.path.join(pkgdatadir, 'runtime', pkg + '.packaged')):
rpkglist.append((rpkg, pkg))
rpkglist.sort(key=lambda x: len(x[0]), reverse=True)
pvu = d.getVar('PV', False)
if '$' + '{SRCPV}' in pvu:
pvprefix = pvu.split('$' + '{SRCPV}', 1)[0]
else:
pvprefix = None
pkgwritetask = 'package_write_%s' % pkgtype
files = []
docopy = False
manifest, _ = oe.sstatesig.sstate_get_manifest_filename(pkgwritetask, d)
mlprefix = d.getVar('MLPREFIX', True)
# Copy recipe's all packages if one of the packages are different to make
# they have the same PR.
with open(manifest, 'r') as f:
for line in f:
if line.startswith(prepath):
srcpath = line.rstrip()
if os.path.isfile(srcpath):
destpath = os.path.join(deploydir, os.path.relpath(srcpath, prepath))
# This is crude but should work assuming the output
# package file name starts with the package name
# and rpkglist is sorted by length (descending)
pkgbasename = os.path.basename(destpath)
pkgname = None
for rpkg, pkg in rpkglist:
if mlprefix and pkgtype == 'rpm' and rpkg.startswith(mlprefix):
rpkg = rpkg[len(mlprefix):]
if pkgbasename.startswith(rpkg):
pkgr = pkgrvalues[pkg]
destpathspec = destpath.replace(pkgr, '*')
if pvprefix:
pkgv = pkgvvalues[pkg]
if pkgv.startswith(pvprefix):
pkgvsuffix = pkgv[len(pvprefix):]
if '+' in pkgvsuffix:
newpkgv = pvprefix + '*+' + pkgvsuffix.split('+', 1)[1]
destpathspec = destpathspec.replace(pkgv, newpkgv)
pkgname = pkg
break
else:
bb.warn('Unable to map %s back to package' % pkgbasename)
destpathspec = destpath
oldfile = None
if not docopy:
oldfiles = glob.glob(destpathspec)
if oldfiles:
oldfile = oldfiles[-1]
result = subprocess.call(['pkg-diff.sh', oldfile, srcpath])
if result != 0:
docopy = True
bb.note("%s and %s are different, will copy packages" % (oldfile, srcpath))
else:
docopy = True
bb.note("No old packages found for %s, will copy packages" % pkgname)
files.append((pkgname, pkgbasename, srcpath, destpath))
# Remove all the old files and copy again if docopy
if docopy:
bb.plain('Copying packages for recipe %s' % pn)
pcmanifest = os.path.join(prepath, d.expand('pkg-compare-manifest-${MULTIMACH_TARGET_SYS}-${PN}'))
try:
with open(pcmanifest, 'r') as f:
for line in f:
fn = line.rstrip()
if fn:
try:
os.remove(fn)
bb.note('Removed old package %s' % fn)
except OSError as e:
if e.errno == errno.ENOENT:
pass
except IOError as e:
if e.errno == errno.ENOENT:
pass
# Create new manifest
with open(pcmanifest, 'w') as f:
for pkgname, pkgbasename, srcpath, destpath in files:
destdir = os.path.dirname(destpath)
bb.utils.mkdirhier(destdir)
# Remove allarch rpm pkg if it is already existed (for
# multilib), they're identical in theory, but sstate.bbclass
# copies it again, so keep align with that.
if os.path.exists(destpath) and pkgtype == 'rpm' \
and d.getVar('PACKAGE_ARCH', True) == 'all':
os.unlink(destpath)
if (os.stat(srcpath).st_dev == os.stat(destdir).st_dev):
# Use a hard link to save space
os.link(srcpath, destpath)
else:
shutil.copyfile(srcpath, destpath)
f.write('%s\n' % destpath)
else:
bb.plain('Not copying packages for recipe %s' % pn)
do_cleansstate[postfuncs] += "pfs_cleanpkgs"
python pfs_cleanpkgs () {
import errno
for pkgclass in (d.getVar('PACKAGE_CLASSES', True) or '').split():
if pkgclass.startswith('package_'):
pkgtype = pkgclass.split('_', 1)[1]
deploydir = d.getVar('DEPLOY_DIR_%s' % pkgtype.upper(), True)
prepath = deploydir + '-prediff'
pcmanifest = os.path.join(prepath, d.expand('pkg-compare-manifest-${MULTIMACH_TARGET_SYS}-${PN}'))
try:
with open(pcmanifest, 'r') as f:
for line in f:
fn = line.rstrip()
if fn:
try:
os.remove(fn)
except OSError as e:
if e.errno == errno.ENOENT:
pass
os.remove(pcmanifest)
except IOError as e:
if e.errno == errno.ENOENT:
pass
}