blob: 2e983d2f175ff856e8b1d1933cd58ccd655aefd1 [file] [log] [blame]
Brad Bishop15ae2502019-06-18 21:44:24 -04001#
2# SPDX-License-Identifier: MIT
3#
Andrew Geissler82c905d2020-04-13 13:39:40 -05004# Copyright 2019-2020 by Garmin Ltd. or its subsidiaries
Brad Bishop15ae2502019-06-18 21:44:24 -04005
6from oeqa.selftest.case import OESelftestTestCase
7from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_bb_vars
Brad Bishop1d80a2e2019-11-15 16:35:03 -05008import bb.utils
Brad Bishop15ae2502019-06-18 21:44:24 -04009import functools
10import multiprocessing
11import textwrap
Brad Bishop79641f22019-09-10 07:20:22 -040012import json
Brad Bishop15ae2502019-06-18 21:44:24 -040013import unittest
Brad Bishop1d80a2e2019-11-15 16:35:03 -050014import tempfile
15import shutil
16import stat
17import os
Andrew Geissler82c905d2020-04-13 13:39:40 -050018import datetime
Brad Bishop15ae2502019-06-18 21:44:24 -040019
Andrew Geisslerd1e89492021-02-12 15:35:20 -060020# For sample packages, see:
21# https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201127-0t7wr_oo/
22# https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201127-4s9ejwyp/
23# https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201127-haiwdlbr/
24# https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201127-hwds3mcl/
25# https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201203-sua0pzvc/
26# (both packages/ and packages-excluded/)
Andrew Geissler9b4d8b02021-02-19 12:26:16 -060027
28# ruby-ri-docs, meson:
29#https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20210215-0_td9la2/packages/diff-html/
Andrew Geisslerd159c7f2021-09-02 21:05:58 -050030# rust-llvm:
31#https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20210825-kaihham6/
Andrew Geisslerd1e89492021-02-12 15:35:20 -060032exclude_packages = [
Andrew Geisslerd1e89492021-02-12 15:35:20 -060033 'glide',
Andrew Geisslerd1e89492021-02-12 15:35:20 -060034 'go-helloworld',
35 'go-runtime',
36 'go_',
Andrew Geissler90fd73c2021-03-05 15:25:55 -060037 'go-',
Andrew Geisslerd159c7f2021-09-02 21:05:58 -050038 'ruby-ri-docs',
39 'rust-llvm-liblto',
40 'rust-llvm-staticdev'
Andrew Geisslerd1e89492021-02-12 15:35:20 -060041 ]
42
43def is_excluded(package):
44 package_name = os.path.basename(package)
45 for i in exclude_packages:
46 if package_name.startswith(i):
Andrew Geissler9b4d8b02021-02-19 12:26:16 -060047 return i
48 return None
Andrew Geisslerd1e89492021-02-12 15:35:20 -060049
Brad Bishop15ae2502019-06-18 21:44:24 -040050MISSING = 'MISSING'
51DIFFERENT = 'DIFFERENT'
52SAME = 'SAME'
53
54@functools.total_ordering
55class CompareResult(object):
56 def __init__(self):
57 self.reference = None
58 self.test = None
59 self.status = 'UNKNOWN'
60
61 def __eq__(self, other):
62 return (self.status, self.test) == (other.status, other.test)
63
64 def __lt__(self, other):
65 return (self.status, self.test) < (other.status, other.test)
66
67class PackageCompareResults(object):
68 def __init__(self):
69 self.total = []
70 self.missing = []
71 self.different = []
Andrew Geisslerd1e89492021-02-12 15:35:20 -060072 self.different_excluded = []
Brad Bishop15ae2502019-06-18 21:44:24 -040073 self.same = []
Andrew Geissler9b4d8b02021-02-19 12:26:16 -060074 self.active_exclusions = set()
Brad Bishop15ae2502019-06-18 21:44:24 -040075
76 def add_result(self, r):
77 self.total.append(r)
78 if r.status == MISSING:
79 self.missing.append(r)
80 elif r.status == DIFFERENT:
Andrew Geissler9b4d8b02021-02-19 12:26:16 -060081 exclusion = is_excluded(r.reference)
82 if exclusion:
Andrew Geisslerd1e89492021-02-12 15:35:20 -060083 self.different_excluded.append(r)
Andrew Geissler9b4d8b02021-02-19 12:26:16 -060084 self.active_exclusions.add(exclusion)
Andrew Geisslerd1e89492021-02-12 15:35:20 -060085 else:
86 self.different.append(r)
Brad Bishop15ae2502019-06-18 21:44:24 -040087 else:
88 self.same.append(r)
89
90 def sort(self):
91 self.total.sort()
92 self.missing.sort()
93 self.different.sort()
Andrew Geisslerd1e89492021-02-12 15:35:20 -060094 self.different_excluded.sort()
Brad Bishop15ae2502019-06-18 21:44:24 -040095 self.same.sort()
96
97 def __str__(self):
Andrew Geissler9b4d8b02021-02-19 12:26:16 -060098 return 'same=%i different=%i different_excluded=%i missing=%i total=%i\nunused_exclusions=%s' % (len(self.same), len(self.different), len(self.different_excluded), len(self.missing), len(self.total), self.unused_exclusions())
99
100 def unused_exclusions(self):
101 return sorted(set(exclude_packages) - self.active_exclusions)
Brad Bishop15ae2502019-06-18 21:44:24 -0400102
103def compare_file(reference, test, diffutils_sysroot):
104 result = CompareResult()
105 result.reference = reference
106 result.test = test
107
108 if not os.path.exists(reference):
109 result.status = MISSING
110 return result
111
Andrew Geissler90fd73c2021-03-05 15:25:55 -0600112 r = runCmd(['cmp', '--quiet', reference, test], native_sysroot=diffutils_sysroot, ignore_status=True, sync=False)
Brad Bishop15ae2502019-06-18 21:44:24 -0400113
114 if r.status:
115 result.status = DIFFERENT
116 return result
117
118 result.status = SAME
119 return result
120
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500121def run_diffoscope(a_dir, b_dir, html_dir, **kwargs):
122 return runCmd(['diffoscope', '--no-default-limits', '--exclude-directory-metadata', 'yes', '--html-dir', html_dir, a_dir, b_dir],
123 **kwargs)
124
125class DiffoscopeTests(OESelftestTestCase):
126 diffoscope_test_files = os.path.join(os.path.dirname(os.path.abspath(__file__)), "diffoscope")
127
128 def test_diffoscope(self):
129 bitbake("diffoscope-native -c addto_recipe_sysroot")
130 diffoscope_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "diffoscope-native")
131
132 # Check that diffoscope doesn't return an error when the files compare
133 # the same (a general check that diffoscope is working)
134 with tempfile.TemporaryDirectory() as tmpdir:
135 run_diffoscope('A', 'A', tmpdir,
136 native_sysroot=diffoscope_sysroot, cwd=self.diffoscope_test_files)
137
138 # Check that diffoscope generates an index.html file when the files are
139 # different
140 with tempfile.TemporaryDirectory() as tmpdir:
141 r = run_diffoscope('A', 'B', tmpdir,
142 native_sysroot=diffoscope_sysroot, ignore_status=True, cwd=self.diffoscope_test_files)
143
144 self.assertNotEqual(r.status, 0, msg="diffoscope was successful when an error was expected")
145 self.assertTrue(os.path.exists(os.path.join(tmpdir, 'index.html')), "HTML index not found!")
146
Brad Bishop15ae2502019-06-18 21:44:24 -0400147class ReproducibleTests(OESelftestTestCase):
Andrew Geissler90fd73c2021-03-05 15:25:55 -0600148 # Test the reproducibility of whatever is built between sstate_targets and targets
149
150 package_classes = ['deb', 'ipk', 'rpm']
151
152 # targets are the things we want to test the reproducibility of
153 targets = ['core-image-minimal', 'core-image-sato', 'core-image-full-cmdline', 'core-image-weston', 'world']
154 # sstate targets are things to pull from sstate to potentially cut build/debugging time
155 sstate_targets = []
Brad Bishop1d80a2e2019-11-15 16:35:03 -0500156 save_results = False
Andrew Geissler82c905d2020-04-13 13:39:40 -0500157 if 'OEQA_DEBUGGING_SAVED_OUTPUT' in os.environ:
158 save_results = os.environ['OEQA_DEBUGGING_SAVED_OUTPUT']
159
160 # This variable controls if one of the test builds is allowed to pull from
161 # an sstate cache/mirror. The other build is always done clean as a point of
162 # comparison.
163 # If you know that your sstate archives are reproducible, enabling this
164 # will test that and also make the test run faster. If your sstate is not
165 # reproducible, disable this in your derived test class
166 build_from_sstate = True
Brad Bishop15ae2502019-06-18 21:44:24 -0400167
168 def setUpLocal(self):
169 super().setUpLocal()
170 needed_vars = ['TOPDIR', 'TARGET_PREFIX', 'BB_NUMBER_THREADS']
171 bb_vars = get_bb_vars(needed_vars)
172 for v in needed_vars:
173 setattr(self, v.lower(), bb_vars[v])
174
Andrew Geissler82c905d2020-04-13 13:39:40 -0500175 self.extraresults = {}
176 self.extraresults.setdefault('reproducible.rawlogs', {})['log'] = ''
177 self.extraresults.setdefault('reproducible', {}).setdefault('files', {})
Brad Bishop15ae2502019-06-18 21:44:24 -0400178
179 def append_to_log(self, msg):
Andrew Geissler82c905d2020-04-13 13:39:40 -0500180 self.extraresults['reproducible.rawlogs']['log'] += msg
Brad Bishop15ae2502019-06-18 21:44:24 -0400181
182 def compare_packages(self, reference_dir, test_dir, diffutils_sysroot):
183 result = PackageCompareResults()
184
185 old_cwd = os.getcwd()
186 try:
187 file_result = {}
188 os.chdir(test_dir)
189 with multiprocessing.Pool(processes=int(self.bb_number_threads or 0)) as p:
190 for root, dirs, files in os.walk('.'):
191 async_result = []
192 for f in files:
193 reference_path = os.path.join(reference_dir, root, f)
194 test_path = os.path.join(test_dir, root, f)
195 async_result.append(p.apply_async(compare_file, (reference_path, test_path, diffutils_sysroot)))
196
197 for a in async_result:
198 result.add_result(a.get())
199
200 finally:
201 os.chdir(old_cwd)
202
203 result.sort()
204 return result
205
Brad Bishop79641f22019-09-10 07:20:22 -0400206 def write_package_list(self, package_class, name, packages):
Andrew Geissler82c905d2020-04-13 13:39:40 -0500207 self.extraresults['reproducible']['files'].setdefault(package_class, {})[name] = [
Brad Bishop79641f22019-09-10 07:20:22 -0400208 {'reference': p.reference, 'test': p.test} for p in packages]
209
Brad Bishop1d80a2e2019-11-15 16:35:03 -0500210 def copy_file(self, source, dest):
211 bb.utils.mkdirhier(os.path.dirname(dest))
212 shutil.copyfile(source, dest)
213
Andrew Geissler82c905d2020-04-13 13:39:40 -0500214 def do_test_build(self, name, use_sstate):
Brad Bishop15ae2502019-06-18 21:44:24 -0400215 capture_vars = ['DEPLOY_DIR_' + c.upper() for c in self.package_classes]
216
Andrew Geissler82c905d2020-04-13 13:39:40 -0500217 tmpdir = os.path.join(self.topdir, name, 'tmp')
218 if os.path.exists(tmpdir):
219 bb.utils.remove(tmpdir, recurse=True)
220
221 config = textwrap.dedent('''\
Andrew Geissler82c905d2020-04-13 13:39:40 -0500222 PACKAGE_CLASSES = "{package_classes}"
223 INHIBIT_PACKAGE_STRIP = "1"
224 TMPDIR = "{tmpdir}"
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600225 LICENSE_FLAGS_WHITELIST = "commercial"
Patrick Williams213cb262021-08-07 19:21:33 -0500226 DISTRO_FEATURES:append = ' systemd pam'
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600227 USERADDEXTENSION = "useradd-staticids"
228 USERADD_ERROR_DYNAMIC = "skip"
229 USERADD_UID_TABLES += "files/static-passwd"
230 USERADD_GID_TABLES += "files/static-group"
Andrew Geissler82c905d2020-04-13 13:39:40 -0500231 ''').format(package_classes=' '.join('package_%s' % c for c in self.package_classes),
232 tmpdir=tmpdir)
233
234 if not use_sstate:
Andrew Geissler90fd73c2021-03-05 15:25:55 -0600235 if self.sstate_targets:
236 self.logger.info("Building prebuild for %s (sstate allowed)..." % (name))
237 self.write_config(config)
238 bitbake(' '.join(self.sstate_targets))
239
Andrew Geissler82c905d2020-04-13 13:39:40 -0500240 # This config fragment will disable using shared and the sstate
241 # mirror, forcing a complete build from scratch
242 config += textwrap.dedent('''\
243 SSTATE_DIR = "${TMPDIR}/sstate"
Andrew Geissler9b4d8b02021-02-19 12:26:16 -0600244 SSTATE_MIRRORS = ""
Andrew Geissler82c905d2020-04-13 13:39:40 -0500245 ''')
246
Andrew Geissler9b4d8b02021-02-19 12:26:16 -0600247 self.logger.info("Building %s (sstate%s allowed)..." % (name, '' if use_sstate else ' NOT'))
Andrew Geissler82c905d2020-04-13 13:39:40 -0500248 self.write_config(config)
249 d = get_bb_vars(capture_vars)
Andrew Geissler90fd73c2021-03-05 15:25:55 -0600250 # targets used to be called images
251 bitbake(' '.join(getattr(self, 'images', self.targets)))
Andrew Geissler82c905d2020-04-13 13:39:40 -0500252 return d
253
254 def test_reproducible_builds(self):
255 def strip_topdir(s):
256 if s.startswith(self.topdir):
257 return s[len(self.topdir):]
258 return s
Brad Bishop1d80a2e2019-11-15 16:35:03 -0500259
Brad Bishop15ae2502019-06-18 21:44:24 -0400260 # Build native utilities
Brad Bishop79641f22019-09-10 07:20:22 -0400261 self.write_config('')
Andrew Geissler82c905d2020-04-13 13:39:40 -0500262 bitbake("diffoscope-native diffutils-native jquery-native -c addto_recipe_sysroot")
Brad Bishop15ae2502019-06-18 21:44:24 -0400263 diffutils_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "diffutils-native")
Andrew Geissler82c905d2020-04-13 13:39:40 -0500264 diffoscope_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "diffoscope-native")
265 jquery_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "jquery-native")
Brad Bishop15ae2502019-06-18 21:44:24 -0400266
Andrew Geissler82c905d2020-04-13 13:39:40 -0500267 if self.save_results:
268 os.makedirs(self.save_results, exist_ok=True)
269 datestr = datetime.datetime.now().strftime('%Y%m%d')
270 save_dir = tempfile.mkdtemp(prefix='oe-reproducible-%s-' % datestr, dir=self.save_results)
271 os.chmod(save_dir, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
272 self.logger.info('Non-reproducible packages will be copied to %s', save_dir)
Brad Bishop79641f22019-09-10 07:20:22 -0400273
Andrew Geissler82c905d2020-04-13 13:39:40 -0500274 vars_A = self.do_test_build('reproducibleA', self.build_from_sstate)
Andrew Geissler9b4d8b02021-02-19 12:26:16 -0600275
Andrew Geissler82c905d2020-04-13 13:39:40 -0500276 vars_B = self.do_test_build('reproducibleB', False)
Brad Bishop79641f22019-09-10 07:20:22 -0400277
278 # NOTE: The temp directories from the reproducible build are purposely
279 # kept after the build so it can be diffed for debugging.
280
Andrew Geissler82c905d2020-04-13 13:39:40 -0500281 fails = []
282
Brad Bishop15ae2502019-06-18 21:44:24 -0400283 for c in self.package_classes:
Brad Bishop79641f22019-09-10 07:20:22 -0400284 with self.subTest(package_class=c):
285 package_class = 'package_' + c
Brad Bishop15ae2502019-06-18 21:44:24 -0400286
Brad Bishop79641f22019-09-10 07:20:22 -0400287 deploy_A = vars_A['DEPLOY_DIR_' + c.upper()]
288 deploy_B = vars_B['DEPLOY_DIR_' + c.upper()]
Brad Bishop15ae2502019-06-18 21:44:24 -0400289
Andrew Geissler9b4d8b02021-02-19 12:26:16 -0600290 self.logger.info('Checking %s packages for differences...' % c)
Brad Bishop79641f22019-09-10 07:20:22 -0400291 result = self.compare_packages(deploy_A, deploy_B, diffutils_sysroot)
Brad Bishop15ae2502019-06-18 21:44:24 -0400292
Brad Bishop79641f22019-09-10 07:20:22 -0400293 self.logger.info('Reproducibility summary for %s: %s' % (c, result))
Brad Bishop15ae2502019-06-18 21:44:24 -0400294
Brad Bishop79641f22019-09-10 07:20:22 -0400295 self.append_to_log('\n'.join("%s: %s" % (r.status, r.test) for r in result.total))
Brad Bishop15ae2502019-06-18 21:44:24 -0400296
Brad Bishop79641f22019-09-10 07:20:22 -0400297 self.write_package_list(package_class, 'missing', result.missing)
298 self.write_package_list(package_class, 'different', result.different)
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600299 self.write_package_list(package_class, 'different_excluded', result.different_excluded)
Brad Bishop79641f22019-09-10 07:20:22 -0400300 self.write_package_list(package_class, 'same', result.same)
301
Brad Bishop1d80a2e2019-11-15 16:35:03 -0500302 if self.save_results:
303 for d in result.different:
Andrew Geissler82c905d2020-04-13 13:39:40 -0500304 self.copy_file(d.reference, '/'.join([save_dir, 'packages', strip_topdir(d.reference)]))
305 self.copy_file(d.test, '/'.join([save_dir, 'packages', strip_topdir(d.test)]))
Brad Bishop1d80a2e2019-11-15 16:35:03 -0500306
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600307 for d in result.different_excluded:
308 self.copy_file(d.reference, '/'.join([save_dir, 'packages-excluded', strip_topdir(d.reference)]))
309 self.copy_file(d.test, '/'.join([save_dir, 'packages-excluded', strip_topdir(d.test)]))
310
Brad Bishop79641f22019-09-10 07:20:22 -0400311 if result.missing or result.different:
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600312 fails.append("The following %s packages are missing or different and not in exclusion list: %s" %
Andrew Geissler82c905d2020-04-13 13:39:40 -0500313 (c, '\n'.join(r.test for r in (result.missing + result.different))))
314
315 # Clean up empty directories
316 if self.save_results:
317 if not os.listdir(save_dir):
318 os.rmdir(save_dir)
319 else:
320 self.logger.info('Running diffoscope')
321 package_dir = os.path.join(save_dir, 'packages')
322 package_html_dir = os.path.join(package_dir, 'diff-html')
323
324 # Copy jquery to improve the diffoscope output usability
325 self.copy_file(os.path.join(jquery_sysroot, 'usr/share/javascript/jquery/jquery.min.js'), os.path.join(package_html_dir, 'jquery.js'))
326
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500327 run_diffoscope('reproducibleA', 'reproducibleB', package_html_dir,
Andrew Geissler82c905d2020-04-13 13:39:40 -0500328 native_sysroot=diffoscope_sysroot, ignore_status=True, cwd=package_dir)
329
330 if fails:
331 self.fail('\n'.join(fails))
Brad Bishop15ae2502019-06-18 21:44:24 -0400332