meta-openembedded and poky: subtree updates
Squash of the following due to dependencies among them
and OpenBMC changes:
meta-openembedded: subtree update:d0748372d2..9201611135
meta-openembedded: subtree update:9201611135..17fd382f34
poky: subtree update:9052e5b32a..2e11d97b6c
poky: subtree update:2e11d97b6c..a8544811d7
The change log was too large for the jenkins plugin
to handle therefore it has been removed. Here is
the first and last commit of each subtree:
meta-openembedded:d0748372d2
cppzmq: bump to version 4.6.0
meta-openembedded:17fd382f34
mpv: Remove X11 dependency
poky:9052e5b32a
package_ipk: Remove pointless comment to trigger rebuild
poky:a8544811d7
pbzip2: Fix license warning
Change-Id: If0fc6c37629642ee207a4ca2f7aa501a2c673cd6
Signed-off-by: Andrew Geissler <geissonator@yahoo.com>
diff --git a/poky/meta/lib/oeqa/selftest/cases/reproducible.py b/poky/meta/lib/oeqa/selftest/cases/reproducible.py
index a911056..5d3959b 100644
--- a/poky/meta/lib/oeqa/selftest/cases/reproducible.py
+++ b/poky/meta/lib/oeqa/selftest/cases/reproducible.py
@@ -1,7 +1,7 @@
#
# SPDX-License-Identifier: MIT
#
-# Copyright 2019 by Garmin Ltd. or its subsidiaries
+# Copyright 2019-2020 by Garmin Ltd. or its subsidiaries
from oeqa.selftest.case import OESelftestTestCase
from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_bb_vars
@@ -15,6 +15,7 @@
import shutil
import stat
import os
+import datetime
MISSING = 'MISSING'
DIFFERENT = 'DIFFERENT'
@@ -78,8 +79,18 @@
class ReproducibleTests(OESelftestTestCase):
package_classes = ['deb', 'ipk']
- images = ['core-image-minimal']
+ images = ['core-image-minimal', 'core-image-sato', 'core-image-full-cmdline']
save_results = False
+ if 'OEQA_DEBUGGING_SAVED_OUTPUT' in os.environ:
+ save_results = os.environ['OEQA_DEBUGGING_SAVED_OUTPUT']
+
+ # This variable controls if one of the test builds is allowed to pull from
+ # an sstate cache/mirror. The other build is always done clean as a point of
+ # comparison.
+ # If you know that your sstate archives are reproducible, enabling this
+ # will test that and also make the test run faster. If your sstate is not
+ # reproducible, disable this in your derived test class
+ build_from_sstate = True
def setUpLocal(self):
super().setUpLocal()
@@ -88,12 +99,12 @@
for v in needed_vars:
setattr(self, v.lower(), bb_vars[v])
- self.extrasresults = {}
- self.extrasresults.setdefault('reproducible.rawlogs', {})['log'] = ''
- self.extrasresults.setdefault('reproducible', {}).setdefault('files', {})
+ self.extraresults = {}
+ self.extraresults.setdefault('reproducible.rawlogs', {})['log'] = ''
+ self.extraresults.setdefault('reproducible', {}).setdefault('files', {})
def append_to_log(self, msg):
- self.extrasresults['reproducible.rawlogs']['log'] += msg
+ self.extraresults['reproducible.rawlogs']['log'] += msg
def compare_packages(self, reference_dir, test_dir, diffutils_sysroot):
result = PackageCompareResults()
@@ -120,60 +131,69 @@
return result
def write_package_list(self, package_class, name, packages):
- self.extrasresults['reproducible']['files'].setdefault(package_class, {})[name] = [
+ self.extraresults['reproducible']['files'].setdefault(package_class, {})[name] = [
{'reference': p.reference, 'test': p.test} for p in packages]
def copy_file(self, source, dest):
bb.utils.mkdirhier(os.path.dirname(dest))
shutil.copyfile(source, dest)
- def test_reproducible_builds(self):
+ def do_test_build(self, name, use_sstate):
capture_vars = ['DEPLOY_DIR_' + c.upper() for c in self.package_classes]
- if self.save_results:
- save_dir = tempfile.mkdtemp(prefix='oe-reproducible-')
- os.chmod(save_dir, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
- self.logger.info('Non-reproducible packages will be copied to %s', save_dir)
+ tmpdir = os.path.join(self.topdir, name, 'tmp')
+ if os.path.exists(tmpdir):
+ bb.utils.remove(tmpdir, recurse=True)
+
+ config = textwrap.dedent('''\
+ INHERIT += "reproducible_build"
+ PACKAGE_CLASSES = "{package_classes}"
+ INHIBIT_PACKAGE_STRIP = "1"
+ TMPDIR = "{tmpdir}"
+ ''').format(package_classes=' '.join('package_%s' % c for c in self.package_classes),
+ tmpdir=tmpdir)
+
+ if not use_sstate:
+ # This config fragment will disable using shared and the sstate
+ # mirror, forcing a complete build from scratch
+ config += textwrap.dedent('''\
+ SSTATE_DIR = "${TMPDIR}/sstate"
+ SSTATE_MIRROR = ""
+ ''')
+
+ self.write_config(config)
+ d = get_bb_vars(capture_vars)
+ bitbake(' '.join(self.images))
+ return d
+
+ def test_reproducible_builds(self):
+ def strip_topdir(s):
+ if s.startswith(self.topdir):
+ return s[len(self.topdir):]
+ return s
# Build native utilities
self.write_config('')
- bitbake("diffutils-native -c addto_recipe_sysroot")
+ bitbake("diffoscope-native diffutils-native jquery-native -c addto_recipe_sysroot")
diffutils_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "diffutils-native")
+ diffoscope_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "diffoscope-native")
+ jquery_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "jquery-native")
- # Reproducible builds should not pull from sstate or mirrors, but
- # sharing DL_DIR is fine
- common_config = textwrap.dedent('''\
- INHERIT += "reproducible_build"
- PACKAGE_CLASSES = "%s"
- SSTATE_DIR = "${TMPDIR}/sstate"
- ''') % (' '.join('package_%s' % c for c in self.package_classes))
+ if self.save_results:
+ os.makedirs(self.save_results, exist_ok=True)
+ datestr = datetime.datetime.now().strftime('%Y%m%d')
+ save_dir = tempfile.mkdtemp(prefix='oe-reproducible-%s-' % datestr, dir=self.save_results)
+ os.chmod(save_dir, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
+ self.logger.info('Non-reproducible packages will be copied to %s', save_dir)
- # Perform a build.
- reproducibleA_tmp = os.path.join(self.topdir, 'reproducibleA', 'tmp')
- if os.path.exists(reproducibleA_tmp):
- bb.utils.remove(reproducibleA_tmp, recurse=True)
-
- self.write_config((textwrap.dedent('''\
- TMPDIR = "%s"
- ''') % reproducibleA_tmp) + common_config)
- vars_A = get_bb_vars(capture_vars)
- bitbake(' '.join(self.images))
-
- # Perform another build.
- reproducibleB_tmp = os.path.join(self.topdir, 'reproducibleB', 'tmp')
- if os.path.exists(reproducibleB_tmp):
- bb.utils.remove(reproducibleB_tmp, recurse=True)
-
- self.write_config((textwrap.dedent('''\
- SSTATE_MIRROR = ""
- TMPDIR = "%s"
- ''') % reproducibleB_tmp) + common_config)
- vars_B = get_bb_vars(capture_vars)
- bitbake(' '.join(self.images))
+ vars_A = self.do_test_build('reproducibleA', self.build_from_sstate)
+ vars_B = self.do_test_build('reproducibleB', False)
# NOTE: The temp directories from the reproducible build are purposely
# kept after the build so it can be diffed for debugging.
+ fails = []
+
for c in self.package_classes:
with self.subTest(package_class=c):
package_class = 'package_' + c
@@ -193,10 +213,28 @@
if self.save_results:
for d in result.different:
- self.copy_file(d.reference, '/'.join([save_dir, d.reference]))
- self.copy_file(d.test, '/'.join([save_dir, d.test]))
+ self.copy_file(d.reference, '/'.join([save_dir, 'packages', strip_topdir(d.reference)]))
+ self.copy_file(d.test, '/'.join([save_dir, 'packages', strip_topdir(d.test)]))
if result.missing or result.different:
- self.fail("The following %s packages are missing or different: %s" %
- (c, ' '.join(r.test for r in (result.missing + result.different))))
+ fails.append("The following %s packages are missing or different: %s" %
+ (c, '\n'.join(r.test for r in (result.missing + result.different))))
+
+ # Clean up empty directories
+ if self.save_results:
+ if not os.listdir(save_dir):
+ os.rmdir(save_dir)
+ else:
+ self.logger.info('Running diffoscope')
+ package_dir = os.path.join(save_dir, 'packages')
+ package_html_dir = os.path.join(package_dir, 'diff-html')
+
+ # Copy jquery to improve the diffoscope output usability
+ self.copy_file(os.path.join(jquery_sysroot, 'usr/share/javascript/jquery/jquery.min.js'), os.path.join(package_html_dir, 'jquery.js'))
+
+ runCmd(['diffoscope', '--no-default-limits', '--exclude-directory-metadata', '--html-dir', package_html_dir, 'reproducibleA', 'reproducibleB'],
+ native_sysroot=diffoscope_sysroot, ignore_status=True, cwd=package_dir)
+
+ if fails:
+ self.fail('\n'.join(fails))