poky/meta/lib/oeqa/selftest/cases/reproducible.py

#
# SPDX-License-Identifier: MIT
#
# Copyright 2019 by Garmin Ltd. or its subsidiaries

from oeqa.selftest.case import OESelftestTestCase
from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_bb_vars
import bb.utils
import functools
import multiprocessing
import textwrap
import json
import unittest
import tempfile
import shutil
import stat
import os

MISSING = 'MISSING'
DIFFERENT = 'DIFFERENT'
SAME = 'SAME'

@functools.total_ordering
class CompareResult(object):
    def __init__(self):
        self.reference = None
        self.test = None
        self.status = 'UNKNOWN'

    def __eq__(self, other):
        return (self.status, self.test) == (other.status, other.test)

    def __lt__(self, other):
        return (self.status, self.test) < (other.status, other.test)

class PackageCompareResults(object):
    def __init__(self):
        self.total = []
        self.missing = []
        self.different = []
        self.same = []

    def add_result(self, r):
        self.total.append(r)
        if r.status == MISSING:
            self.missing.append(r)
        elif r.status == DIFFERENT:
            self.different.append(r)
        else:
            self.same.append(r)

    def sort(self):
        self.total.sort()
        self.missing.sort()
        self.different.sort()
        self.same.sort()

    def __str__(self):
        return 'same=%i different=%i missing=%i total=%i' % (len(self.same), len(self.different), len(self.missing), len(self.total))

def compare_file(reference, test, diffutils_sysroot):
    result = CompareResult()
    result.reference = reference
    result.test = test

    if not os.path.exists(reference):
        result.status = MISSING
        return result

    r = runCmd(['cmp', '--quiet', reference, test], native_sysroot=diffutils_sysroot, ignore_status=True)

    if r.status:
        result.status = DIFFERENT
        return result

    result.status = SAME
    return result

class ReproducibleTests(OESelftestTestCase):
    package_classes = ['deb', 'ipk']
    images = ['core-image-minimal']
    save_results = False

    def setUpLocal(self):
        super().setUpLocal()
        needed_vars = ['TOPDIR', 'TARGET_PREFIX', 'BB_NUMBER_THREADS']
        bb_vars = get_bb_vars(needed_vars)
        for v in needed_vars:
            setattr(self, v.lower(), bb_vars[v])

        self.extrasresults = {}
        self.extrasresults.setdefault('reproducible.rawlogs', {})['log'] = ''
        self.extrasresults.setdefault('reproducible', {}).setdefault('files', {})

    def append_to_log(self, msg):
        self.extrasresults['reproducible.rawlogs']['log'] += msg

    def compare_packages(self, reference_dir, test_dir, diffutils_sysroot):
        result = PackageCompareResults()

        old_cwd = os.getcwd()
        try:
            file_result = {}
            os.chdir(test_dir)
            with multiprocessing.Pool(processes=int(self.bb_number_threads or 0)) as p:
                for root, dirs, files in os.walk('.'):
                    async_result = []
                    for f in files:
                        reference_path = os.path.join(reference_dir, root, f)
                        test_path = os.path.join(test_dir, root, f)
                        async_result.append(p.apply_async(compare_file, (reference_path, test_path, diffutils_sysroot)))

                    for a in async_result:
                        result.add_result(a.get())

        finally:
            os.chdir(old_cwd)

        result.sort()
        return result

    def write_package_list(self, package_class, name, packages):
        self.extrasresults['reproducible']['files'].setdefault(package_class, {})[name] = [
                {'reference': p.reference, 'test': p.test} for p in packages]

    def copy_file(self, source, dest):
        bb.utils.mkdirhier(os.path.dirname(dest))
        shutil.copyfile(source, dest)

    def test_reproducible_builds(self):
        capture_vars = ['DEPLOY_DIR_' + c.upper() for c in self.package_classes]

        if self.save_results:
            save_dir = tempfile.mkdtemp(prefix='oe-reproducible-')
            os.chmod(save_dir, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
            self.logger.info('Non-reproducible packages will be copied to %s', save_dir)

        # Build native utilities
        self.write_config('')
        bitbake("diffutils-native -c addto_recipe_sysroot")
        diffutils_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "diffutils-native")

        # Reproducible builds should not pull from sstate or mirrors, but
        # sharing DL_DIR is fine
        common_config = textwrap.dedent('''\
            INHERIT += "reproducible_build"
            PACKAGE_CLASSES = "%s"
            SSTATE_DIR = "${TMPDIR}/sstate"
            ''') % (' '.join('package_%s' % c for c in self.package_classes))

        # Perform a build.
        reproducibleA_tmp = os.path.join(self.topdir, 'reproducibleA', 'tmp')
        if os.path.exists(reproducibleA_tmp):
            bb.utils.remove(reproducibleA_tmp, recurse=True)

        self.write_config((textwrap.dedent('''\
            TMPDIR = "%s"
            ''') % reproducibleA_tmp) + common_config)
        vars_A = get_bb_vars(capture_vars)
        bitbake(' '.join(self.images))

        # Perform another build.
        reproducibleB_tmp = os.path.join(self.topdir, 'reproducibleB', 'tmp')
        if os.path.exists(reproducibleB_tmp):
            bb.utils.remove(reproducibleB_tmp, recurse=True)

        self.write_config((textwrap.dedent('''\
            SSTATE_MIRROR = ""
            TMPDIR = "%s"
            ''') % reproducibleB_tmp) + common_config)
        vars_B = get_bb_vars(capture_vars)
        bitbake(' '.join(self.images))

        # NOTE: The temp directories from the reproducible build are purposely
        # kept after the build so it can be diffed for debugging.

        for c in self.package_classes:
            with self.subTest(package_class=c):
                package_class = 'package_' + c

                deploy_A = vars_A['DEPLOY_DIR_' + c.upper()]
                deploy_B = vars_B['DEPLOY_DIR_' + c.upper()]

                result = self.compare_packages(deploy_A, deploy_B, diffutils_sysroot)

                self.logger.info('Reproducibility summary for %s: %s' % (c, result))

                self.append_to_log('\n'.join("%s: %s" % (r.status, r.test) for r in result.total))

                self.write_package_list(package_class, 'missing', result.missing)
                self.write_package_list(package_class, 'different', result.different)
                self.write_package_list(package_class, 'same', result.same)

                if self.save_results:
                    for d in result.different:
                        self.copy_file(d.reference, '/'.join([save_dir, d.reference]))
                        self.copy_file(d.test, '/'.join([save_dir, d.test]))

                if result.missing or result.different:
                    self.fail("The following %s packages are missing or different: %s" %
                            (c, ' '.join(r.test for r in (result.missing + result.different))))