Brad Bishop | 15ae250 | 2019-06-18 21:44:24 -0400 | [diff] [blame] | 1 | # |
| 2 | # SPDX-License-Identifier: MIT |
| 3 | # |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 4 | # Copyright 2019-2020 by Garmin Ltd. or its subsidiaries |
Brad Bishop | 15ae250 | 2019-06-18 21:44:24 -0400 | [diff] [blame] | 5 | |
| 6 | from oeqa.selftest.case import OESelftestTestCase |
| 7 | from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_bb_vars |
Brad Bishop | 1d80a2e | 2019-11-15 16:35:03 -0500 | [diff] [blame] | 8 | import bb.utils |
Brad Bishop | 15ae250 | 2019-06-18 21:44:24 -0400 | [diff] [blame] | 9 | import functools |
| 10 | import multiprocessing |
| 11 | import textwrap |
Brad Bishop | 1d80a2e | 2019-11-15 16:35:03 -0500 | [diff] [blame] | 12 | import tempfile |
| 13 | import shutil |
| 14 | import stat |
| 15 | import os |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 16 | import datetime |
Brad Bishop | 15ae250 | 2019-06-18 21:44:24 -0400 | [diff] [blame] | 17 | |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 18 | exclude_packages = [ |
Patrick Williams | da29531 | 2023-12-05 16:48:56 -0600 | [diff] [blame^] | 19 | 'rust-rustdoc', |
Patrick Williams | ac13d5f | 2023-11-24 18:59:46 -0600 | [diff] [blame] | 20 | 'rust-dbg' |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 21 | ] |
| 22 | |
| 23 | def is_excluded(package): |
| 24 | package_name = os.path.basename(package) |
| 25 | for i in exclude_packages: |
| 26 | if package_name.startswith(i): |
Andrew Geissler | 9b4d8b0 | 2021-02-19 12:26:16 -0600 | [diff] [blame] | 27 | return i |
| 28 | return None |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 29 | |
Brad Bishop | 15ae250 | 2019-06-18 21:44:24 -0400 | [diff] [blame] | 30 | MISSING = 'MISSING' |
| 31 | DIFFERENT = 'DIFFERENT' |
| 32 | SAME = 'SAME' |
| 33 | |
| 34 | @functools.total_ordering |
| 35 | class CompareResult(object): |
| 36 | def __init__(self): |
| 37 | self.reference = None |
| 38 | self.test = None |
| 39 | self.status = 'UNKNOWN' |
| 40 | |
| 41 | def __eq__(self, other): |
| 42 | return (self.status, self.test) == (other.status, other.test) |
| 43 | |
| 44 | def __lt__(self, other): |
| 45 | return (self.status, self.test) < (other.status, other.test) |
| 46 | |
| 47 | class PackageCompareResults(object): |
Patrick Williams | ac13d5f | 2023-11-24 18:59:46 -0600 | [diff] [blame] | 48 | def __init__(self, exclusions): |
Brad Bishop | 15ae250 | 2019-06-18 21:44:24 -0400 | [diff] [blame] | 49 | self.total = [] |
| 50 | self.missing = [] |
| 51 | self.different = [] |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 52 | self.different_excluded = [] |
Brad Bishop | 15ae250 | 2019-06-18 21:44:24 -0400 | [diff] [blame] | 53 | self.same = [] |
Andrew Geissler | 9b4d8b0 | 2021-02-19 12:26:16 -0600 | [diff] [blame] | 54 | self.active_exclusions = set() |
Patrick Williams | ac13d5f | 2023-11-24 18:59:46 -0600 | [diff] [blame] | 55 | exclude_packages.extend((exclusions or "").split()) |
Brad Bishop | 15ae250 | 2019-06-18 21:44:24 -0400 | [diff] [blame] | 56 | |
| 57 | def add_result(self, r): |
| 58 | self.total.append(r) |
| 59 | if r.status == MISSING: |
| 60 | self.missing.append(r) |
| 61 | elif r.status == DIFFERENT: |
Andrew Geissler | 9b4d8b0 | 2021-02-19 12:26:16 -0600 | [diff] [blame] | 62 | exclusion = is_excluded(r.reference) |
| 63 | if exclusion: |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 64 | self.different_excluded.append(r) |
Andrew Geissler | 9b4d8b0 | 2021-02-19 12:26:16 -0600 | [diff] [blame] | 65 | self.active_exclusions.add(exclusion) |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 66 | else: |
| 67 | self.different.append(r) |
Brad Bishop | 15ae250 | 2019-06-18 21:44:24 -0400 | [diff] [blame] | 68 | else: |
| 69 | self.same.append(r) |
| 70 | |
| 71 | def sort(self): |
| 72 | self.total.sort() |
| 73 | self.missing.sort() |
| 74 | self.different.sort() |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 75 | self.different_excluded.sort() |
Brad Bishop | 15ae250 | 2019-06-18 21:44:24 -0400 | [diff] [blame] | 76 | self.same.sort() |
| 77 | |
| 78 | def __str__(self): |
Andrew Geissler | 9b4d8b0 | 2021-02-19 12:26:16 -0600 | [diff] [blame] | 79 | return 'same=%i different=%i different_excluded=%i missing=%i total=%i\nunused_exclusions=%s' % (len(self.same), len(self.different), len(self.different_excluded), len(self.missing), len(self.total), self.unused_exclusions()) |
| 80 | |
| 81 | def unused_exclusions(self): |
| 82 | return sorted(set(exclude_packages) - self.active_exclusions) |
Brad Bishop | 15ae250 | 2019-06-18 21:44:24 -0400 | [diff] [blame] | 83 | |
| 84 | def compare_file(reference, test, diffutils_sysroot): |
| 85 | result = CompareResult() |
| 86 | result.reference = reference |
| 87 | result.test = test |
| 88 | |
| 89 | if not os.path.exists(reference): |
| 90 | result.status = MISSING |
| 91 | return result |
| 92 | |
Andrew Geissler | 90fd73c | 2021-03-05 15:25:55 -0600 | [diff] [blame] | 93 | r = runCmd(['cmp', '--quiet', reference, test], native_sysroot=diffutils_sysroot, ignore_status=True, sync=False) |
Brad Bishop | 15ae250 | 2019-06-18 21:44:24 -0400 | [diff] [blame] | 94 | |
| 95 | if r.status: |
| 96 | result.status = DIFFERENT |
| 97 | return result |
| 98 | |
| 99 | result.status = SAME |
| 100 | return result |
| 101 | |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 102 | def run_diffoscope(a_dir, b_dir, html_dir, max_report_size=0, **kwargs): |
| 103 | return runCmd(['diffoscope', '--no-default-limits', '--max-report-size', str(max_report_size), |
| 104 | '--exclude-directory-metadata', 'yes', '--html-dir', html_dir, a_dir, b_dir], |
Andrew Geissler | c9f7865 | 2020-09-18 14:11:35 -0500 | [diff] [blame] | 105 | **kwargs) |
| 106 | |
| 107 | class DiffoscopeTests(OESelftestTestCase): |
| 108 | diffoscope_test_files = os.path.join(os.path.dirname(os.path.abspath(__file__)), "diffoscope") |
| 109 | |
| 110 | def test_diffoscope(self): |
| 111 | bitbake("diffoscope-native -c addto_recipe_sysroot") |
| 112 | diffoscope_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "diffoscope-native") |
| 113 | |
| 114 | # Check that diffoscope doesn't return an error when the files compare |
| 115 | # the same (a general check that diffoscope is working) |
| 116 | with tempfile.TemporaryDirectory() as tmpdir: |
| 117 | run_diffoscope('A', 'A', tmpdir, |
| 118 | native_sysroot=diffoscope_sysroot, cwd=self.diffoscope_test_files) |
| 119 | |
| 120 | # Check that diffoscope generates an index.html file when the files are |
| 121 | # different |
| 122 | with tempfile.TemporaryDirectory() as tmpdir: |
| 123 | r = run_diffoscope('A', 'B', tmpdir, |
| 124 | native_sysroot=diffoscope_sysroot, ignore_status=True, cwd=self.diffoscope_test_files) |
| 125 | |
| 126 | self.assertNotEqual(r.status, 0, msg="diffoscope was successful when an error was expected") |
| 127 | self.assertTrue(os.path.exists(os.path.join(tmpdir, 'index.html')), "HTML index not found!") |
| 128 | |
Brad Bishop | 15ae250 | 2019-06-18 21:44:24 -0400 | [diff] [blame] | 129 | class ReproducibleTests(OESelftestTestCase): |
Andrew Geissler | 90fd73c | 2021-03-05 15:25:55 -0600 | [diff] [blame] | 130 | # Test the reproducibility of whatever is built between sstate_targets and targets |
| 131 | |
Andrew Geissler | 220dafd | 2023-10-04 10:18:08 -0500 | [diff] [blame] | 132 | package_classes = ['deb', 'ipk', 'rpm'] |
Andrew Geissler | 90fd73c | 2021-03-05 15:25:55 -0600 | [diff] [blame] | 133 | |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 134 | # Maximum report size, in bytes |
| 135 | max_report_size = 250 * 1024 * 1024 |
| 136 | |
Andrew Geissler | 90fd73c | 2021-03-05 15:25:55 -0600 | [diff] [blame] | 137 | # targets are the things we want to test the reproducibility of |
Andrew Geissler | 220dafd | 2023-10-04 10:18:08 -0500 | [diff] [blame] | 138 | targets = ['core-image-minimal', 'core-image-sato', 'core-image-full-cmdline', 'core-image-weston', 'world'] |
| 139 | |
Andrew Geissler | 90fd73c | 2021-03-05 15:25:55 -0600 | [diff] [blame] | 140 | # sstate targets are things to pull from sstate to potentially cut build/debugging time |
Andrew Geissler | 220dafd | 2023-10-04 10:18:08 -0500 | [diff] [blame] | 141 | sstate_targets = [] |
| 142 | |
Brad Bishop | 1d80a2e | 2019-11-15 16:35:03 -0500 | [diff] [blame] | 143 | save_results = False |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 144 | if 'OEQA_DEBUGGING_SAVED_OUTPUT' in os.environ: |
| 145 | save_results = os.environ['OEQA_DEBUGGING_SAVED_OUTPUT'] |
| 146 | |
| 147 | # This variable controls if one of the test builds is allowed to pull from |
| 148 | # an sstate cache/mirror. The other build is always done clean as a point of |
| 149 | # comparison. |
| 150 | # If you know that your sstate archives are reproducible, enabling this |
| 151 | # will test that and also make the test run faster. If your sstate is not |
| 152 | # reproducible, disable this in your derived test class |
| 153 | build_from_sstate = True |
Brad Bishop | 15ae250 | 2019-06-18 21:44:24 -0400 | [diff] [blame] | 154 | |
| 155 | def setUpLocal(self): |
| 156 | super().setUpLocal() |
Patrick Williams | ac13d5f | 2023-11-24 18:59:46 -0600 | [diff] [blame] | 157 | needed_vars = [ |
| 158 | 'TOPDIR', |
| 159 | 'TARGET_PREFIX', |
| 160 | 'BB_NUMBER_THREADS', |
| 161 | 'BB_HASHSERVE', |
| 162 | 'OEQA_REPRODUCIBLE_TEST_PACKAGE', |
| 163 | 'OEQA_REPRODUCIBLE_TEST_TARGET', |
| 164 | 'OEQA_REPRODUCIBLE_TEST_SSTATE_TARGETS', |
| 165 | 'OEQA_REPRODUCIBLE_EXCLUDED_PACKAGES', |
| 166 | ] |
Brad Bishop | 15ae250 | 2019-06-18 21:44:24 -0400 | [diff] [blame] | 167 | bb_vars = get_bb_vars(needed_vars) |
| 168 | for v in needed_vars: |
| 169 | setattr(self, v.lower(), bb_vars[v]) |
| 170 | |
Andrew Geissler | 220dafd | 2023-10-04 10:18:08 -0500 | [diff] [blame] | 171 | if bb_vars['OEQA_REPRODUCIBLE_TEST_PACKAGE']: |
| 172 | self.package_classes = bb_vars['OEQA_REPRODUCIBLE_TEST_PACKAGE'].split() |
| 173 | |
| 174 | if bb_vars['OEQA_REPRODUCIBLE_TEST_TARGET']: |
| 175 | self.targets = bb_vars['OEQA_REPRODUCIBLE_TEST_TARGET'].split() |
| 176 | |
| 177 | if bb_vars['OEQA_REPRODUCIBLE_TEST_SSTATE_TARGETS']: |
| 178 | self.sstate_targets = bb_vars['OEQA_REPRODUCIBLE_TEST_SSTATE_TARGETS'].split() |
| 179 | |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 180 | self.extraresults = {} |
| 181 | self.extraresults.setdefault('reproducible.rawlogs', {})['log'] = '' |
| 182 | self.extraresults.setdefault('reproducible', {}).setdefault('files', {}) |
Brad Bishop | 15ae250 | 2019-06-18 21:44:24 -0400 | [diff] [blame] | 183 | |
| 184 | def append_to_log(self, msg): |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 185 | self.extraresults['reproducible.rawlogs']['log'] += msg |
Brad Bishop | 15ae250 | 2019-06-18 21:44:24 -0400 | [diff] [blame] | 186 | |
| 187 | def compare_packages(self, reference_dir, test_dir, diffutils_sysroot): |
Patrick Williams | ac13d5f | 2023-11-24 18:59:46 -0600 | [diff] [blame] | 188 | result = PackageCompareResults(self.oeqa_reproducible_excluded_packages) |
Brad Bishop | 15ae250 | 2019-06-18 21:44:24 -0400 | [diff] [blame] | 189 | |
| 190 | old_cwd = os.getcwd() |
| 191 | try: |
| 192 | file_result = {} |
| 193 | os.chdir(test_dir) |
| 194 | with multiprocessing.Pool(processes=int(self.bb_number_threads or 0)) as p: |
| 195 | for root, dirs, files in os.walk('.'): |
| 196 | async_result = [] |
| 197 | for f in files: |
| 198 | reference_path = os.path.join(reference_dir, root, f) |
| 199 | test_path = os.path.join(test_dir, root, f) |
| 200 | async_result.append(p.apply_async(compare_file, (reference_path, test_path, diffutils_sysroot))) |
| 201 | |
| 202 | for a in async_result: |
| 203 | result.add_result(a.get()) |
| 204 | |
| 205 | finally: |
| 206 | os.chdir(old_cwd) |
| 207 | |
| 208 | result.sort() |
| 209 | return result |
| 210 | |
Brad Bishop | 79641f2 | 2019-09-10 07:20:22 -0400 | [diff] [blame] | 211 | def write_package_list(self, package_class, name, packages): |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 212 | self.extraresults['reproducible']['files'].setdefault(package_class, {})[name] = [ |
Brad Bishop | 79641f2 | 2019-09-10 07:20:22 -0400 | [diff] [blame] | 213 | {'reference': p.reference, 'test': p.test} for p in packages] |
| 214 | |
Brad Bishop | 1d80a2e | 2019-11-15 16:35:03 -0500 | [diff] [blame] | 215 | def copy_file(self, source, dest): |
| 216 | bb.utils.mkdirhier(os.path.dirname(dest)) |
| 217 | shutil.copyfile(source, dest) |
| 218 | |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 219 | def do_test_build(self, name, use_sstate): |
Brad Bishop | 15ae250 | 2019-06-18 21:44:24 -0400 | [diff] [blame] | 220 | capture_vars = ['DEPLOY_DIR_' + c.upper() for c in self.package_classes] |
| 221 | |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 222 | tmpdir = os.path.join(self.topdir, name, 'tmp') |
| 223 | if os.path.exists(tmpdir): |
| 224 | bb.utils.remove(tmpdir, recurse=True) |
| 225 | |
| 226 | config = textwrap.dedent('''\ |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 227 | PACKAGE_CLASSES = "{package_classes}" |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 228 | TMPDIR = "{tmpdir}" |
Andrew Geissler | 7e0e3c0 | 2022-02-25 20:34:39 +0000 | [diff] [blame] | 229 | LICENSE_FLAGS_ACCEPTED = "commercial" |
Patrick Williams | 2a25492 | 2023-08-11 09:48:11 -0500 | [diff] [blame] | 230 | DISTRO_FEATURES:append = ' pam' |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 231 | USERADDEXTENSION = "useradd-staticids" |
| 232 | USERADD_ERROR_DYNAMIC = "skip" |
| 233 | USERADD_UID_TABLES += "files/static-passwd" |
| 234 | USERADD_GID_TABLES += "files/static-group" |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 235 | ''').format(package_classes=' '.join('package_%s' % c for c in self.package_classes), |
| 236 | tmpdir=tmpdir) |
| 237 | |
| 238 | if not use_sstate: |
Andrew Geissler | 90fd73c | 2021-03-05 15:25:55 -0600 | [diff] [blame] | 239 | if self.sstate_targets: |
| 240 | self.logger.info("Building prebuild for %s (sstate allowed)..." % (name)) |
| 241 | self.write_config(config) |
| 242 | bitbake(' '.join(self.sstate_targets)) |
| 243 | |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 244 | # This config fragment will disable using shared and the sstate |
| 245 | # mirror, forcing a complete build from scratch |
| 246 | config += textwrap.dedent('''\ |
| 247 | SSTATE_DIR = "${TMPDIR}/sstate" |
Patrick Williams | b542dec | 2023-06-09 01:26:37 -0500 | [diff] [blame] | 248 | SSTATE_MIRRORS = "file://.*/.*-native.* http://sstate.yoctoproject.org/all/PATH;downloadfilename=PATH file://.*/.*-cross.* http://sstate.yoctoproject.org/all/PATH;downloadfilename=PATH" |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 249 | ''') |
| 250 | |
Andrew Geissler | 9b4d8b0 | 2021-02-19 12:26:16 -0600 | [diff] [blame] | 251 | self.logger.info("Building %s (sstate%s allowed)..." % (name, '' if use_sstate else ' NOT')) |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 252 | self.write_config(config) |
| 253 | d = get_bb_vars(capture_vars) |
Andrew Geissler | 90fd73c | 2021-03-05 15:25:55 -0600 | [diff] [blame] | 254 | # targets used to be called images |
| 255 | bitbake(' '.join(getattr(self, 'images', self.targets))) |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 256 | return d |
| 257 | |
| 258 | def test_reproducible_builds(self): |
| 259 | def strip_topdir(s): |
| 260 | if s.startswith(self.topdir): |
| 261 | return s[len(self.topdir):] |
| 262 | return s |
Brad Bishop | 1d80a2e | 2019-11-15 16:35:03 -0500 | [diff] [blame] | 263 | |
Brad Bishop | 15ae250 | 2019-06-18 21:44:24 -0400 | [diff] [blame] | 264 | # Build native utilities |
Brad Bishop | 79641f2 | 2019-09-10 07:20:22 -0400 | [diff] [blame] | 265 | self.write_config('') |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 266 | bitbake("diffoscope-native diffutils-native jquery-native -c addto_recipe_sysroot") |
Brad Bishop | 15ae250 | 2019-06-18 21:44:24 -0400 | [diff] [blame] | 267 | diffutils_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "diffutils-native") |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 268 | diffoscope_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "diffoscope-native") |
| 269 | jquery_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "jquery-native") |
Brad Bishop | 15ae250 | 2019-06-18 21:44:24 -0400 | [diff] [blame] | 270 | |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 271 | if self.save_results: |
| 272 | os.makedirs(self.save_results, exist_ok=True) |
| 273 | datestr = datetime.datetime.now().strftime('%Y%m%d') |
| 274 | save_dir = tempfile.mkdtemp(prefix='oe-reproducible-%s-' % datestr, dir=self.save_results) |
| 275 | os.chmod(save_dir, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) |
| 276 | self.logger.info('Non-reproducible packages will be copied to %s', save_dir) |
Brad Bishop | 79641f2 | 2019-09-10 07:20:22 -0400 | [diff] [blame] | 277 | |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 278 | vars_A = self.do_test_build('reproducibleA', self.build_from_sstate) |
Andrew Geissler | 9b4d8b0 | 2021-02-19 12:26:16 -0600 | [diff] [blame] | 279 | |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 280 | vars_B = self.do_test_build('reproducibleB', False) |
Brad Bishop | 79641f2 | 2019-09-10 07:20:22 -0400 | [diff] [blame] | 281 | |
| 282 | # NOTE: The temp directories from the reproducible build are purposely |
| 283 | # kept after the build so it can be diffed for debugging. |
| 284 | |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 285 | fails = [] |
| 286 | |
Brad Bishop | 15ae250 | 2019-06-18 21:44:24 -0400 | [diff] [blame] | 287 | for c in self.package_classes: |
Brad Bishop | 79641f2 | 2019-09-10 07:20:22 -0400 | [diff] [blame] | 288 | with self.subTest(package_class=c): |
| 289 | package_class = 'package_' + c |
Brad Bishop | 15ae250 | 2019-06-18 21:44:24 -0400 | [diff] [blame] | 290 | |
Brad Bishop | 79641f2 | 2019-09-10 07:20:22 -0400 | [diff] [blame] | 291 | deploy_A = vars_A['DEPLOY_DIR_' + c.upper()] |
| 292 | deploy_B = vars_B['DEPLOY_DIR_' + c.upper()] |
Brad Bishop | 15ae250 | 2019-06-18 21:44:24 -0400 | [diff] [blame] | 293 | |
Andrew Geissler | 9b4d8b0 | 2021-02-19 12:26:16 -0600 | [diff] [blame] | 294 | self.logger.info('Checking %s packages for differences...' % c) |
Brad Bishop | 79641f2 | 2019-09-10 07:20:22 -0400 | [diff] [blame] | 295 | result = self.compare_packages(deploy_A, deploy_B, diffutils_sysroot) |
Brad Bishop | 15ae250 | 2019-06-18 21:44:24 -0400 | [diff] [blame] | 296 | |
Brad Bishop | 79641f2 | 2019-09-10 07:20:22 -0400 | [diff] [blame] | 297 | self.logger.info('Reproducibility summary for %s: %s' % (c, result)) |
Brad Bishop | 15ae250 | 2019-06-18 21:44:24 -0400 | [diff] [blame] | 298 | |
Brad Bishop | 79641f2 | 2019-09-10 07:20:22 -0400 | [diff] [blame] | 299 | self.append_to_log('\n'.join("%s: %s" % (r.status, r.test) for r in result.total)) |
Brad Bishop | 15ae250 | 2019-06-18 21:44:24 -0400 | [diff] [blame] | 300 | |
Brad Bishop | 79641f2 | 2019-09-10 07:20:22 -0400 | [diff] [blame] | 301 | self.write_package_list(package_class, 'missing', result.missing) |
| 302 | self.write_package_list(package_class, 'different', result.different) |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 303 | self.write_package_list(package_class, 'different_excluded', result.different_excluded) |
Brad Bishop | 79641f2 | 2019-09-10 07:20:22 -0400 | [diff] [blame] | 304 | self.write_package_list(package_class, 'same', result.same) |
| 305 | |
Brad Bishop | 1d80a2e | 2019-11-15 16:35:03 -0500 | [diff] [blame] | 306 | if self.save_results: |
| 307 | for d in result.different: |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 308 | self.copy_file(d.reference, '/'.join([save_dir, 'packages', strip_topdir(d.reference)])) |
| 309 | self.copy_file(d.test, '/'.join([save_dir, 'packages', strip_topdir(d.test)])) |
Brad Bishop | 1d80a2e | 2019-11-15 16:35:03 -0500 | [diff] [blame] | 310 | |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 311 | for d in result.different_excluded: |
| 312 | self.copy_file(d.reference, '/'.join([save_dir, 'packages-excluded', strip_topdir(d.reference)])) |
| 313 | self.copy_file(d.test, '/'.join([save_dir, 'packages-excluded', strip_topdir(d.test)])) |
| 314 | |
Andrew Geissler | fc113ea | 2023-03-31 09:59:46 -0500 | [diff] [blame] | 315 | if result.different: |
| 316 | fails.append("The following %s packages are different and not in exclusion list:\n%s" % |
| 317 | (c, '\n'.join(r.test for r in (result.different)))) |
| 318 | |
| 319 | if result.missing and len(self.sstate_targets) == 0: |
| 320 | fails.append("The following %s packages are missing and not in exclusion list:\n%s" % |
| 321 | (c, '\n'.join(r.test for r in (result.missing)))) |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 322 | |
| 323 | # Clean up empty directories |
| 324 | if self.save_results: |
| 325 | if not os.listdir(save_dir): |
| 326 | os.rmdir(save_dir) |
| 327 | else: |
| 328 | self.logger.info('Running diffoscope') |
| 329 | package_dir = os.path.join(save_dir, 'packages') |
| 330 | package_html_dir = os.path.join(package_dir, 'diff-html') |
| 331 | |
| 332 | # Copy jquery to improve the diffoscope output usability |
| 333 | self.copy_file(os.path.join(jquery_sysroot, 'usr/share/javascript/jquery/jquery.min.js'), os.path.join(package_html_dir, 'jquery.js')) |
| 334 | |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 335 | run_diffoscope('reproducibleA', 'reproducibleB', package_html_dir, max_report_size=self.max_report_size, |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 336 | native_sysroot=diffoscope_sysroot, ignore_status=True, cwd=package_dir) |
| 337 | |
| 338 | if fails: |
| 339 | self.fail('\n'.join(fails)) |
Brad Bishop | 15ae250 | 2019-06-18 21:44:24 -0400 | [diff] [blame] | 340 | |