poky/scripts/contrib/image-manifest

#!/usr/bin/env python3

# Script to extract information from image manifests
#
# Copyright (C) 2018 Intel Corporation
# Copyright (C) 2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: GPL-2.0-only
#

import sys
import os
import argparse
import logging
import json
import shutil
import tempfile
import tarfile
from collections import OrderedDict

scripts_path = os.path.dirname(__file__)
lib_path = scripts_path + '/../lib'
sys.path = sys.path + [lib_path]

import scriptutils
logger = scriptutils.logger_create(os.path.basename(__file__))

import argparse_oe
import scriptpath
bitbakepath = scriptpath.add_bitbake_lib_path()
if not bitbakepath:
    logger.error("Unable to find bitbake by searching parent directory of this script or PATH")
    sys.exit(1)
logger.debug('Using standard bitbake path %s' % bitbakepath)
scriptpath.add_oe_lib_path()

import bb.tinfoil
import bb.utils
import oe.utils
import oe.recipeutils

def get_pkg_list(manifest):
    pkglist = []
    with open(manifest, 'r') as f:
        for line in f:
            linesplit = line.split()
            if len(linesplit) == 3:
                # manifest file
                pkglist.append(linesplit[0])
            elif len(linesplit) == 1:
                # build dependency file
                pkglist.append(linesplit[0])
    return sorted(pkglist)

def list_packages(args):
    pkglist = get_pkg_list(args.manifest)
    for pkg in pkglist:
        print('%s' % pkg)

def pkg2recipe(tinfoil, pkg):
    if "-native" in pkg:
        logger.info('skipping %s' % pkg)
        return None

    pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR')
    pkgdatafile = os.path.join(pkgdata_dir, 'runtime-reverse', pkg)
    logger.debug('pkgdatafile %s' % pkgdatafile)
    try:
        f = open(pkgdatafile, 'r')
        for line in f:
            if line.startswith('PN:'):
                recipe = line.split(':', 1)[1].strip()
                return recipe
    except Exception:
        logger.warning('%s is missing' % pkgdatafile)
        return None

def get_recipe_list(manifest, tinfoil):
    pkglist = get_pkg_list(manifest)
    recipelist = []
    for pkg in pkglist:
        recipe = pkg2recipe(tinfoil,pkg)
        if recipe:
            if not recipe in recipelist:
                recipelist.append(recipe)

    return sorted(recipelist)

def list_recipes(args):
    import bb.tinfoil
    with bb.tinfoil.Tinfoil() as tinfoil:
        tinfoil.logger.setLevel(logger.getEffectiveLevel())
        tinfoil.prepare(config_only=True)
        recipelist = get_recipe_list(args.manifest, tinfoil)
        for recipe in sorted(recipelist):
            print('%s' % recipe)

def list_layers(args):

    def find_git_repo(pth):
        checkpth = pth
        while checkpth != os.sep:
            if os.path.exists(os.path.join(checkpth, '.git')):
                return checkpth
            checkpth = os.path.dirname(checkpth)
        return None

    def get_git_remote_branch(repodir):
        try:
            stdout, _ = bb.process.run(['git', 'rev-parse', '--abbrev-ref', '--symbolic-full-name', '@{u}'], cwd=repodir)
        except bb.process.ExecutionError as e:
            stdout = None
        if stdout:
            return stdout.strip()
        else:
            return None

    def get_git_head_commit(repodir):
        try:
            stdout, _ = bb.process.run(['git', 'rev-parse', 'HEAD'], cwd=repodir)
        except bb.process.ExecutionError as e:
            stdout = None
        if stdout:
            return stdout.strip()
        else:
            return None

    def get_git_repo_url(repodir, remote='origin'):
        import bb.process
        # Try to get upstream repo location from origin remote
        try:
            stdout, _ = bb.process.run(['git', 'remote', '-v'], cwd=repodir)
        except bb.process.ExecutionError as e:
            stdout = None
        if stdout:
            for line in stdout.splitlines():
                splitline = line.split()
                if len(splitline) > 1:
                    if splitline[0] == remote and scriptutils.is_src_url(splitline[1]):
                        return splitline[1]
        return None

    with bb.tinfoil.Tinfoil() as tinfoil:
        tinfoil.logger.setLevel(logger.getEffectiveLevel())
        tinfoil.prepare(config_only=False)
        layers = OrderedDict()
        for layerdir in tinfoil.config_data.getVar('BBLAYERS').split():
            layerdata = OrderedDict()
            layername = os.path.basename(layerdir)
            logger.debug('layername %s, layerdir %s' % (layername, layerdir))
            if layername in layers:
                logger.warning('layername %s is not unique in configuration' % layername)
                layername = os.path.basename(os.path.dirname(layerdir)) + '_' + os.path.basename(layerdir)
                logger.debug('trying layername %s' % layername)
                if layername in layers:
                    logger.error('Layer name %s is not unique in configuration' % layername)
                    sys.exit(2)
            repodir = find_git_repo(layerdir)
            if repodir:
                remotebranch = get_git_remote_branch(repodir)
                remote = 'origin'
                if remotebranch and '/' in remotebranch:
                    rbsplit = remotebranch.split('/', 1)
                    layerdata['actual_branch'] = rbsplit[1]
                    remote = rbsplit[0]
                layerdata['vcs_url'] = get_git_repo_url(repodir, remote)
                if os.path.abspath(repodir) != os.path.abspath(layerdir):
                    layerdata['vcs_subdir'] = os.path.relpath(layerdir, repodir)
                commit = get_git_head_commit(repodir)
                if commit:
                    layerdata['vcs_commit'] = commit
            layers[layername] = layerdata

    json.dump(layers, args.output, indent=2)

def get_recipe(args):
    with bb.tinfoil.Tinfoil() as tinfoil:
        tinfoil.logger.setLevel(logger.getEffectiveLevel())
        tinfoil.prepare(config_only=True)

        recipe = pkg2recipe(tinfoil, args.package)
        print(' %s package provided by %s' % (args.package, recipe))

def pkg_dependencies(args):
    def get_recipe_info(tinfoil, recipe):
        try:
            info = tinfoil.get_recipe_info(recipe)
        except Exception:
            logger.error('Failed to get recipe info for: %s' % recipe)
            sys.exit(1)
        if not info:
            logger.warning('No recipe info found for: %s' % recipe)
            sys.exit(1)
        append_files = tinfoil.get_file_appends(info.fn)
        appends = True
        data = tinfoil.parse_recipe_file(info.fn, appends, append_files)
        data.pn = info.pn
        data.pv = info.pv
        return data

    def find_dependencies(tinfoil, assume_provided, recipe_info, packages, rn, order):
        spaces = '  ' * order
        data = recipe_info[rn]
        if args.native:
            logger.debug('%s- %s' % (spaces, data.pn))
        elif "-native" not in data.pn:
            if "cross" not in data.pn:
                logger.debug('%s- %s' % (spaces, data.pn))

        depends = []
        for dep in data.depends:
            if dep not in assume_provided:
                depends.append(dep)

        # First find all dependencies not in package list.
        for dep in depends:
            if dep not in packages:
                packages.append(dep)
                dep_data = get_recipe_info(tinfoil, dep)
                # Do this once now to reduce the number of bitbake calls.
                dep_data.depends = dep_data.getVar('DEPENDS').split()
                recipe_info[dep] = dep_data

        # Then recursively analyze all of the dependencies for the current recipe.
        for dep in depends:
            find_dependencies(tinfoil, assume_provided, recipe_info, packages, dep, order + 1)

    with bb.tinfoil.Tinfoil() as tinfoil:
        tinfoil.logger.setLevel(logger.getEffectiveLevel())
        tinfoil.prepare()

        assume_provided = tinfoil.config_data.getVar('ASSUME_PROVIDED').split()
        logger.debug('assumed provided:')
        for ap in sorted(assume_provided):
            logger.debug(' - %s' % ap)

        recipe = pkg2recipe(tinfoil, args.package)
        data = get_recipe_info(tinfoil, recipe)
        data.depends = []
        depends = data.getVar('DEPENDS').split()
        for dep in depends:
            if dep not in assume_provided:
                data.depends.append(dep)

        recipe_info = dict([(recipe, data)])
        packages = []
        find_dependencies(tinfoil, assume_provided, recipe_info, packages, recipe, order=1)

        print('\nThe following packages are required to build %s' % recipe)
        for p in sorted(packages):
            data = recipe_info[p]
            if "-native" not in data.pn:
                if "cross" not in data.pn:
                    print(" %s (%s)" % (data.pn,p))

        if args.native:
            print('\nThe following native packages are required to build %s' % recipe)
            for p in sorted(packages):
                data = recipe_info[p]
                if "-native" in data.pn:
                    print(" %s(%s)" % (data.pn,p))
                if "cross" in data.pn:
                    print(" %s(%s)" % (data.pn,p))

def default_config():
    vlist = OrderedDict()
    vlist['PV'] = 'yes'
    vlist['SUMMARY'] = 'no'
    vlist['DESCRIPTION'] = 'no'
    vlist['SECTION'] = 'no'
    vlist['LICENSE'] = 'yes'
    vlist['HOMEPAGE'] = 'no'
    vlist['BUGTRACKER'] = 'no'
    vlist['PROVIDES'] = 'no'
    vlist['BBCLASSEXTEND'] = 'no'
    vlist['DEPENDS'] = 'no'
    vlist['PACKAGECONFIG'] = 'no'
    vlist['SRC_URI'] = 'yes'
    vlist['SRCREV'] = 'yes'
    vlist['EXTRA_OECONF'] = 'no'
    vlist['EXTRA_OESCONS'] = 'no'
    vlist['EXTRA_OECMAKE'] = 'no'
    vlist['EXTRA_OEMESON'] = 'no'

    clist = OrderedDict()
    clist['variables'] = vlist
    clist['filepath'] = 'no'
    clist['sha256sum'] = 'no'
    clist['layerdir'] = 'no'
    clist['layer'] = 'no'
    clist['inherits'] = 'no'
    clist['source_urls'] = 'no'
    clist['packageconfig_opts'] = 'no'
    clist['patches'] = 'no'
    clist['packagedir'] = 'no'
    return clist

def dump_config(args):
    config = default_config()
    f = open('default_config.json', 'w')
    json.dump(config, f, indent=2)
    logger.info('Default config list dumped to default_config.json')

def export_manifest_info(args):

    def handle_value(value):
        if value:
            return oe.utils.squashspaces(value)
        else:
            return value

    if args.config:
        logger.debug('config: %s' % args.config)
        f = open(args.config, 'r')
        config = json.load(f, object_pairs_hook=OrderedDict)
    else:
        config = default_config()
    if logger.isEnabledFor(logging.DEBUG):
        print('Configuration:')
        json.dump(config, sys.stdout, indent=2)
        print('')

    tmpoutdir = tempfile.mkdtemp(prefix=os.path.basename(__file__)+'-')
    logger.debug('tmp dir: %s' % tmpoutdir)

    # export manifest
    shutil.copy2(args.manifest,os.path.join(tmpoutdir, "manifest"))

    with bb.tinfoil.Tinfoil(tracking=True) as tinfoil:
        tinfoil.logger.setLevel(logger.getEffectiveLevel())
        tinfoil.prepare(config_only=False)

        pkglist = get_pkg_list(args.manifest)
        # export pkg list
        f = open(os.path.join(tmpoutdir, "pkgs"), 'w')
        for pkg in pkglist:
            f.write('%s\n' % pkg)
        f.close()

        recipelist = []
        for pkg in pkglist:
            recipe = pkg2recipe(tinfoil,pkg)
            if recipe:
                if not recipe in recipelist:
                    recipelist.append(recipe)
        recipelist.sort()
        # export recipe list
        f = open(os.path.join(tmpoutdir, "recipes"), 'w')
        for recipe in recipelist:
            f.write('%s\n' % recipe)
        f.close()

        try:
            rvalues = OrderedDict()
            for pn in sorted(recipelist):
                logger.debug('Package: %s' % pn)
                rd = tinfoil.parse_recipe(pn)

                rvalues[pn] = OrderedDict()

                for varname in config['variables']:
                    if config['variables'][varname] == 'yes':
                        rvalues[pn][varname] = handle_value(rd.getVar(varname))

                fpth = rd.getVar('FILE')
                layerdir = oe.recipeutils.find_layerdir(fpth)
                if config['filepath'] == 'yes':
                    rvalues[pn]['filepath'] = os.path.relpath(fpth, layerdir)
                    if config['sha256sum'] == 'yes':
                        rvalues[pn]['sha256sum'] = bb.utils.sha256_file(fpth)

                if config['layerdir'] == 'yes':
                    rvalues[pn]['layerdir'] = layerdir

                if config['layer'] == 'yes':
                    rvalues[pn]['layer'] = os.path.basename(layerdir)

                if config['inherits'] == 'yes':
                    gr = set(tinfoil.config_data.getVar("__inherit_cache") or [])
                    lr = set(rd.getVar("__inherit_cache") or [])
                    rvalues[pn]['inherits'] = sorted({os.path.splitext(os.path.basename(r))[0] for r in lr if r not in gr})

                if config['source_urls'] == 'yes':
                    rvalues[pn]['source_urls'] = []
                    for url in (rd.getVar('SRC_URI') or '').split():
                        if not url.startswith('file://'):
                            url = url.split(';')[0]
                            rvalues[pn]['source_urls'].append(url)

                if config['packageconfig_opts'] == 'yes':
                    rvalues[pn]['packageconfig_opts'] = OrderedDict()
                    for key in rd.getVarFlags('PACKAGECONFIG').keys():
                        if key == 'doc':
                            continue
                        rvalues[pn]['packageconfig_opts'][key] = rd.getVarFlag('PACKAGECONFIG', key)

                if config['patches'] == 'yes':
                    patches = oe.recipeutils.get_recipe_patches(rd)
                    rvalues[pn]['patches'] = []
                    if patches:
                        recipeoutdir = os.path.join(tmpoutdir, pn, 'patches')
                        bb.utils.mkdirhier(recipeoutdir)
                        for patch in patches:
                            # Patches may be in other layers too
                            patchlayerdir = oe.recipeutils.find_layerdir(patch)
                            # patchlayerdir will be None for remote patches, which we ignore
                            # (since currently they are considered as part of sources)
                            if patchlayerdir:
                                rvalues[pn]['patches'].append((os.path.basename(patchlayerdir), os.path.relpath(patch, patchlayerdir)))
                                shutil.copy(patch, recipeoutdir)

                if config['packagedir'] == 'yes':
                    pn_dir = os.path.join(tmpoutdir, pn)
                    bb.utils.mkdirhier(pn_dir)
                    f = open(os.path.join(pn_dir, 'recipe.json'), 'w')
                    json.dump(rvalues[pn], f, indent=2)
                    f.close()

            with open(os.path.join(tmpoutdir, 'recipes.json'), 'w') as f:
                json.dump(rvalues, f, indent=2)

            if args.output:
                outname = os.path.basename(args.output)
            else:
                outname = os.path.splitext(os.path.basename(args.manifest))[0]
            if outname.endswith('.tar.gz'):
                outname = outname[:-7]
            elif outname.endswith('.tgz'):
                outname = outname[:-4]

            tarfn = outname
            if tarfn.endswith(os.sep):
                tarfn = tarfn[:-1]
            if not tarfn.endswith(('.tar.gz', '.tgz')):
                tarfn += '.tar.gz'
            with open(tarfn, 'wb') as f:
                with tarfile.open(None, "w:gz", f) as tar:
                    tar.add(tmpoutdir, outname)
        finally:
            shutil.rmtree(tmpoutdir)


def main():
    parser = argparse_oe.ArgumentParser(description="Image manifest utility",
                                        epilog="Use %(prog)s <subcommand> --help to get help on a specific command")
    parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true')
    parser.add_argument('-q', '--quiet', help='Print only errors', action='store_true')
    subparsers = parser.add_subparsers(dest="subparser_name", title='subcommands', metavar='<subcommand>')
    subparsers.required = True

    # get recipe info
    parser_get_recipes = subparsers.add_parser('recipe-info',
                                          help='Get recipe info',
                                          description='Get recipe information for a package')
    parser_get_recipes.add_argument('package', help='Package name')
    parser_get_recipes.set_defaults(func=get_recipe)

    # list runtime dependencies
    parser_pkg_dep = subparsers.add_parser('list-depends',
                                          help='List dependencies',
                                          description='List dependencies required to build the package')
    parser_pkg_dep.add_argument('--native', help='also print native  and cross packages', action='store_true')
    parser_pkg_dep.add_argument('package', help='Package name')
    parser_pkg_dep.set_defaults(func=pkg_dependencies)

    # list recipes
    parser_recipes = subparsers.add_parser('list-recipes',
                                          help='List recipes producing packages within an image',
                                          description='Lists recipes producing the packages that went into an image, using the manifest and pkgdata')
    parser_recipes.add_argument('manifest', help='Manifest file')
    parser_recipes.set_defaults(func=list_recipes)

    # list packages
    parser_packages = subparsers.add_parser('list-packages',
                                          help='List packages within an image',
                                          description='Lists packages that went into an image, using the manifest')
    parser_packages.add_argument('manifest', help='Manifest file')
    parser_packages.set_defaults(func=list_packages)

    # list layers
    parser_layers = subparsers.add_parser('list-layers',
                                          help='List included layers',
                                          description='Lists included layers')
    parser_layers.add_argument('-o', '--output', help='Output file - defaults to stdout if not specified',
                                default=sys.stdout, type=argparse.FileType('w'))
    parser_layers.set_defaults(func=list_layers)

    # dump default configuration file
    parser_dconfig = subparsers.add_parser('dump-config',
                                          help='Dump default config',
                                          description='Dump default config to default_config.json')
    parser_dconfig.set_defaults(func=dump_config)

    # export recipe info for packages in manifest
    parser_export = subparsers.add_parser('manifest-info',
                                          help='Export recipe info for a manifest',
                                          description='Export recipe information using the manifest')
    parser_export.add_argument('-c', '--config', help='load config from json file')
    parser_export.add_argument('-o', '--output', help='Output file (tarball) - defaults to manifest name if not specified')
    parser_export.add_argument('manifest', help='Manifest file')
    parser_export.set_defaults(func=export_manifest_info)

    args = parser.parse_args()

    if args.debug:
        logger.setLevel(logging.DEBUG)
        logger.debug("Debug Enabled")
    elif args.quiet:
        logger.setLevel(logging.ERROR)

    ret = args.func(args)

    return ret


if __name__ == "__main__":
    try:
        ret = main()
    except Exception:
        ret = 1
        import traceback
        traceback.print_exc()
    sys.exit(ret)