| #!/usr/bin/env python3 |
| |
| # Script to extract information from image manifests |
| # |
| # Copyright (C) 2018 Intel Corporation |
| # Copyright (C) 2021 Wind River Systems, Inc. |
| # |
| # SPDX-License-Identifier: GPL-2.0-only |
| # |
| |
| import sys |
| import os |
| import argparse |
| import logging |
| import json |
| import shutil |
| import tempfile |
| import tarfile |
| from collections import OrderedDict |
| |
| scripts_path = os.path.dirname(__file__) |
| lib_path = scripts_path + '/../lib' |
| sys.path = sys.path + [lib_path] |
| |
| import scriptutils |
| logger = scriptutils.logger_create(os.path.basename(__file__)) |
| |
| import argparse_oe |
| import scriptpath |
| bitbakepath = scriptpath.add_bitbake_lib_path() |
| if not bitbakepath: |
| logger.error("Unable to find bitbake by searching parent directory of this script or PATH") |
| sys.exit(1) |
| logger.debug('Using standard bitbake path %s' % bitbakepath) |
| scriptpath.add_oe_lib_path() |
| |
| import bb.tinfoil |
| import bb.utils |
| import oe.utils |
| import oe.recipeutils |
| |
| def get_pkg_list(manifest): |
| pkglist = [] |
| with open(manifest, 'r') as f: |
| for line in f: |
| linesplit = line.split() |
| if len(linesplit) == 3: |
| # manifest file |
| pkglist.append(linesplit[0]) |
| elif len(linesplit) == 1: |
| # build dependency file |
| pkglist.append(linesplit[0]) |
| return sorted(pkglist) |
| |
| def list_packages(args): |
| pkglist = get_pkg_list(args.manifest) |
| for pkg in pkglist: |
| print('%s' % pkg) |
| |
| def pkg2recipe(tinfoil, pkg): |
| if "-native" in pkg: |
| logger.info('skipping %s' % pkg) |
| return None |
| |
| pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR') |
| pkgdatafile = os.path.join(pkgdata_dir, 'runtime-reverse', pkg) |
| logger.debug('pkgdatafile %s' % pkgdatafile) |
| try: |
| f = open(pkgdatafile, 'r') |
| for line in f: |
| if line.startswith('PN:'): |
| recipe = line.split(':', 1)[1].strip() |
| return recipe |
| except Exception: |
| logger.warning('%s is missing' % pkgdatafile) |
| return None |
| |
| def get_recipe_list(manifest, tinfoil): |
| pkglist = get_pkg_list(manifest) |
| recipelist = [] |
| for pkg in pkglist: |
| recipe = pkg2recipe(tinfoil,pkg) |
| if recipe: |
| if not recipe in recipelist: |
| recipelist.append(recipe) |
| |
| return sorted(recipelist) |
| |
| def list_recipes(args): |
| import bb.tinfoil |
| with bb.tinfoil.Tinfoil() as tinfoil: |
| tinfoil.logger.setLevel(logger.getEffectiveLevel()) |
| tinfoil.prepare(config_only=True) |
| recipelist = get_recipe_list(args.manifest, tinfoil) |
| for recipe in sorted(recipelist): |
| print('%s' % recipe) |
| |
| def list_layers(args): |
| |
| def find_git_repo(pth): |
| checkpth = pth |
| while checkpth != os.sep: |
| if os.path.exists(os.path.join(checkpth, '.git')): |
| return checkpth |
| checkpth = os.path.dirname(checkpth) |
| return None |
| |
| def get_git_remote_branch(repodir): |
| try: |
| stdout, _ = bb.process.run(['git', 'rev-parse', '--abbrev-ref', '--symbolic-full-name', '@{u}'], cwd=repodir) |
| except bb.process.ExecutionError as e: |
| stdout = None |
| if stdout: |
| return stdout.strip() |
| else: |
| return None |
| |
| def get_git_head_commit(repodir): |
| try: |
| stdout, _ = bb.process.run(['git', 'rev-parse', 'HEAD'], cwd=repodir) |
| except bb.process.ExecutionError as e: |
| stdout = None |
| if stdout: |
| return stdout.strip() |
| else: |
| return None |
| |
| def get_git_repo_url(repodir, remote='origin'): |
| import bb.process |
| # Try to get upstream repo location from origin remote |
| try: |
| stdout, _ = bb.process.run(['git', 'remote', '-v'], cwd=repodir) |
| except bb.process.ExecutionError as e: |
| stdout = None |
| if stdout: |
| for line in stdout.splitlines(): |
| splitline = line.split() |
| if len(splitline) > 1: |
| if splitline[0] == remote and scriptutils.is_src_url(splitline[1]): |
| return splitline[1] |
| return None |
| |
| with bb.tinfoil.Tinfoil() as tinfoil: |
| tinfoil.logger.setLevel(logger.getEffectiveLevel()) |
| tinfoil.prepare(config_only=False) |
| layers = OrderedDict() |
| for layerdir in tinfoil.config_data.getVar('BBLAYERS').split(): |
| layerdata = OrderedDict() |
| layername = os.path.basename(layerdir) |
| logger.debug('layername %s, layerdir %s' % (layername, layerdir)) |
| if layername in layers: |
| logger.warning('layername %s is not unique in configuration' % layername) |
| layername = os.path.basename(os.path.dirname(layerdir)) + '_' + os.path.basename(layerdir) |
| logger.debug('trying layername %s' % layername) |
| if layername in layers: |
| logger.error('Layer name %s is not unique in configuration' % layername) |
| sys.exit(2) |
| repodir = find_git_repo(layerdir) |
| if repodir: |
| remotebranch = get_git_remote_branch(repodir) |
| remote = 'origin' |
| if remotebranch and '/' in remotebranch: |
| rbsplit = remotebranch.split('/', 1) |
| layerdata['actual_branch'] = rbsplit[1] |
| remote = rbsplit[0] |
| layerdata['vcs_url'] = get_git_repo_url(repodir, remote) |
| if os.path.abspath(repodir) != os.path.abspath(layerdir): |
| layerdata['vcs_subdir'] = os.path.relpath(layerdir, repodir) |
| commit = get_git_head_commit(repodir) |
| if commit: |
| layerdata['vcs_commit'] = commit |
| layers[layername] = layerdata |
| |
| json.dump(layers, args.output, indent=2) |
| |
| def get_recipe(args): |
| with bb.tinfoil.Tinfoil() as tinfoil: |
| tinfoil.logger.setLevel(logger.getEffectiveLevel()) |
| tinfoil.prepare(config_only=True) |
| |
| recipe = pkg2recipe(tinfoil, args.package) |
| print(' %s package provided by %s' % (args.package, recipe)) |
| |
| def pkg_dependencies(args): |
| def get_recipe_info(tinfoil, recipe): |
| try: |
| info = tinfoil.get_recipe_info(recipe) |
| except Exception: |
| logger.error('Failed to get recipe info for: %s' % recipe) |
| sys.exit(1) |
| if not info: |
| logger.warning('No recipe info found for: %s' % recipe) |
| sys.exit(1) |
| append_files = tinfoil.get_file_appends(info.fn) |
| appends = True |
| data = tinfoil.parse_recipe_file(info.fn, appends, append_files) |
| data.pn = info.pn |
| data.pv = info.pv |
| return data |
| |
| def find_dependencies(tinfoil, assume_provided, recipe_info, packages, rn, order): |
| spaces = ' ' * order |
| data = recipe_info[rn] |
| if args.native: |
| logger.debug('%s- %s' % (spaces, data.pn)) |
| elif "-native" not in data.pn: |
| if "cross" not in data.pn: |
| logger.debug('%s- %s' % (spaces, data.pn)) |
| |
| depends = [] |
| for dep in data.depends: |
| if dep not in assume_provided: |
| depends.append(dep) |
| |
| # First find all dependencies not in package list. |
| for dep in depends: |
| if dep not in packages: |
| packages.append(dep) |
| dep_data = get_recipe_info(tinfoil, dep) |
| # Do this once now to reduce the number of bitbake calls. |
| dep_data.depends = dep_data.getVar('DEPENDS').split() |
| recipe_info[dep] = dep_data |
| |
| # Then recursively analyze all of the dependencies for the current recipe. |
| for dep in depends: |
| find_dependencies(tinfoil, assume_provided, recipe_info, packages, dep, order + 1) |
| |
| with bb.tinfoil.Tinfoil() as tinfoil: |
| tinfoil.logger.setLevel(logger.getEffectiveLevel()) |
| tinfoil.prepare() |
| |
| assume_provided = tinfoil.config_data.getVar('ASSUME_PROVIDED').split() |
| logger.debug('assumed provided:') |
| for ap in sorted(assume_provided): |
| logger.debug(' - %s' % ap) |
| |
| recipe = pkg2recipe(tinfoil, args.package) |
| data = get_recipe_info(tinfoil, recipe) |
| data.depends = [] |
| depends = data.getVar('DEPENDS').split() |
| for dep in depends: |
| if dep not in assume_provided: |
| data.depends.append(dep) |
| |
| recipe_info = dict([(recipe, data)]) |
| packages = [] |
| find_dependencies(tinfoil, assume_provided, recipe_info, packages, recipe, order=1) |
| |
| print('\nThe following packages are required to build %s' % recipe) |
| for p in sorted(packages): |
| data = recipe_info[p] |
| if "-native" not in data.pn: |
| if "cross" not in data.pn: |
| print(" %s (%s)" % (data.pn,p)) |
| |
| if args.native: |
| print('\nThe following native packages are required to build %s' % recipe) |
| for p in sorted(packages): |
| data = recipe_info[p] |
| if "-native" in data.pn: |
| print(" %s(%s)" % (data.pn,p)) |
| if "cross" in data.pn: |
| print(" %s(%s)" % (data.pn,p)) |
| |
| def default_config(): |
| vlist = OrderedDict() |
| vlist['PV'] = 'yes' |
| vlist['SUMMARY'] = 'no' |
| vlist['DESCRIPTION'] = 'no' |
| vlist['SECTION'] = 'no' |
| vlist['LICENSE'] = 'yes' |
| vlist['HOMEPAGE'] = 'no' |
| vlist['BUGTRACKER'] = 'no' |
| vlist['PROVIDES'] = 'no' |
| vlist['BBCLASSEXTEND'] = 'no' |
| vlist['DEPENDS'] = 'no' |
| vlist['PACKAGECONFIG'] = 'no' |
| vlist['SRC_URI'] = 'yes' |
| vlist['SRCREV'] = 'yes' |
| vlist['EXTRA_OECONF'] = 'no' |
| vlist['EXTRA_OESCONS'] = 'no' |
| vlist['EXTRA_OECMAKE'] = 'no' |
| vlist['EXTRA_OEMESON'] = 'no' |
| |
| clist = OrderedDict() |
| clist['variables'] = vlist |
| clist['filepath'] = 'no' |
| clist['sha256sum'] = 'no' |
| clist['layerdir'] = 'no' |
| clist['layer'] = 'no' |
| clist['inherits'] = 'no' |
| clist['source_urls'] = 'no' |
| clist['packageconfig_opts'] = 'no' |
| clist['patches'] = 'no' |
| clist['packagedir'] = 'no' |
| return clist |
| |
| def dump_config(args): |
| config = default_config() |
| f = open('default_config.json', 'w') |
| json.dump(config, f, indent=2) |
| logger.info('Default config list dumped to default_config.json') |
| |
| def export_manifest_info(args): |
| |
| def handle_value(value): |
| if value: |
| return oe.utils.squashspaces(value) |
| else: |
| return value |
| |
| if args.config: |
| logger.debug('config: %s' % args.config) |
| f = open(args.config, 'r') |
| config = json.load(f, object_pairs_hook=OrderedDict) |
| else: |
| config = default_config() |
| if logger.isEnabledFor(logging.DEBUG): |
| print('Configuration:') |
| json.dump(config, sys.stdout, indent=2) |
| print('') |
| |
| tmpoutdir = tempfile.mkdtemp(prefix=os.path.basename(__file__)+'-') |
| logger.debug('tmp dir: %s' % tmpoutdir) |
| |
| # export manifest |
| shutil.copy2(args.manifest,os.path.join(tmpoutdir, "manifest")) |
| |
| with bb.tinfoil.Tinfoil(tracking=True) as tinfoil: |
| tinfoil.logger.setLevel(logger.getEffectiveLevel()) |
| tinfoil.prepare(config_only=False) |
| |
| pkglist = get_pkg_list(args.manifest) |
| # export pkg list |
| f = open(os.path.join(tmpoutdir, "pkgs"), 'w') |
| for pkg in pkglist: |
| f.write('%s\n' % pkg) |
| f.close() |
| |
| recipelist = [] |
| for pkg in pkglist: |
| recipe = pkg2recipe(tinfoil,pkg) |
| if recipe: |
| if not recipe in recipelist: |
| recipelist.append(recipe) |
| recipelist.sort() |
| # export recipe list |
| f = open(os.path.join(tmpoutdir, "recipes"), 'w') |
| for recipe in recipelist: |
| f.write('%s\n' % recipe) |
| f.close() |
| |
| try: |
| rvalues = OrderedDict() |
| for pn in sorted(recipelist): |
| logger.debug('Package: %s' % pn) |
| rd = tinfoil.parse_recipe(pn) |
| |
| rvalues[pn] = OrderedDict() |
| |
| for varname in config['variables']: |
| if config['variables'][varname] == 'yes': |
| rvalues[pn][varname] = handle_value(rd.getVar(varname)) |
| |
| fpth = rd.getVar('FILE') |
| layerdir = oe.recipeutils.find_layerdir(fpth) |
| if config['filepath'] == 'yes': |
| rvalues[pn]['filepath'] = os.path.relpath(fpth, layerdir) |
| if config['sha256sum'] == 'yes': |
| rvalues[pn]['sha256sum'] = bb.utils.sha256_file(fpth) |
| |
| if config['layerdir'] == 'yes': |
| rvalues[pn]['layerdir'] = layerdir |
| |
| if config['layer'] == 'yes': |
| rvalues[pn]['layer'] = os.path.basename(layerdir) |
| |
| if config['inherits'] == 'yes': |
| gr = set(tinfoil.config_data.getVar("__inherit_cache") or []) |
| lr = set(rd.getVar("__inherit_cache") or []) |
| rvalues[pn]['inherits'] = sorted({os.path.splitext(os.path.basename(r))[0] for r in lr if r not in gr}) |
| |
| if config['source_urls'] == 'yes': |
| rvalues[pn]['source_urls'] = [] |
| for url in (rd.getVar('SRC_URI') or '').split(): |
| if not url.startswith('file://'): |
| url = url.split(';')[0] |
| rvalues[pn]['source_urls'].append(url) |
| |
| if config['packageconfig_opts'] == 'yes': |
| rvalues[pn]['packageconfig_opts'] = OrderedDict() |
| for key in rd.getVarFlags('PACKAGECONFIG').keys(): |
| if key == 'doc': |
| continue |
| rvalues[pn]['packageconfig_opts'][key] = rd.getVarFlag('PACKAGECONFIG', key) |
| |
| if config['patches'] == 'yes': |
| patches = oe.recipeutils.get_recipe_patches(rd) |
| rvalues[pn]['patches'] = [] |
| if patches: |
| recipeoutdir = os.path.join(tmpoutdir, pn, 'patches') |
| bb.utils.mkdirhier(recipeoutdir) |
| for patch in patches: |
| # Patches may be in other layers too |
| patchlayerdir = oe.recipeutils.find_layerdir(patch) |
| # patchlayerdir will be None for remote patches, which we ignore |
| # (since currently they are considered as part of sources) |
| if patchlayerdir: |
| rvalues[pn]['patches'].append((os.path.basename(patchlayerdir), os.path.relpath(patch, patchlayerdir))) |
| shutil.copy(patch, recipeoutdir) |
| |
| if config['packagedir'] == 'yes': |
| pn_dir = os.path.join(tmpoutdir, pn) |
| bb.utils.mkdirhier(pn_dir) |
| f = open(os.path.join(pn_dir, 'recipe.json'), 'w') |
| json.dump(rvalues[pn], f, indent=2) |
| f.close() |
| |
| with open(os.path.join(tmpoutdir, 'recipes.json'), 'w') as f: |
| json.dump(rvalues, f, indent=2) |
| |
| if args.output: |
| outname = os.path.basename(args.output) |
| else: |
| outname = os.path.splitext(os.path.basename(args.manifest))[0] |
| if outname.endswith('.tar.gz'): |
| outname = outname[:-7] |
| elif outname.endswith('.tgz'): |
| outname = outname[:-4] |
| |
| tarfn = outname |
| if tarfn.endswith(os.sep): |
| tarfn = tarfn[:-1] |
| if not tarfn.endswith(('.tar.gz', '.tgz')): |
| tarfn += '.tar.gz' |
| with open(tarfn, 'wb') as f: |
| with tarfile.open(None, "w:gz", f) as tar: |
| tar.add(tmpoutdir, outname) |
| finally: |
| shutil.rmtree(tmpoutdir) |
| |
| |
| def main(): |
| parser = argparse_oe.ArgumentParser(description="Image manifest utility", |
| epilog="Use %(prog)s <subcommand> --help to get help on a specific command") |
| parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true') |
| parser.add_argument('-q', '--quiet', help='Print only errors', action='store_true') |
| subparsers = parser.add_subparsers(dest="subparser_name", title='subcommands', metavar='<subcommand>') |
| subparsers.required = True |
| |
| # get recipe info |
| parser_get_recipes = subparsers.add_parser('recipe-info', |
| help='Get recipe info', |
| description='Get recipe information for a package') |
| parser_get_recipes.add_argument('package', help='Package name') |
| parser_get_recipes.set_defaults(func=get_recipe) |
| |
| # list runtime dependencies |
| parser_pkg_dep = subparsers.add_parser('list-depends', |
| help='List dependencies', |
| description='List dependencies required to build the package') |
| parser_pkg_dep.add_argument('--native', help='also print native and cross packages', action='store_true') |
| parser_pkg_dep.add_argument('package', help='Package name') |
| parser_pkg_dep.set_defaults(func=pkg_dependencies) |
| |
| # list recipes |
| parser_recipes = subparsers.add_parser('list-recipes', |
| help='List recipes producing packages within an image', |
| description='Lists recipes producing the packages that went into an image, using the manifest and pkgdata') |
| parser_recipes.add_argument('manifest', help='Manifest file') |
| parser_recipes.set_defaults(func=list_recipes) |
| |
| # list packages |
| parser_packages = subparsers.add_parser('list-packages', |
| help='List packages within an image', |
| description='Lists packages that went into an image, using the manifest') |
| parser_packages.add_argument('manifest', help='Manifest file') |
| parser_packages.set_defaults(func=list_packages) |
| |
| # list layers |
| parser_layers = subparsers.add_parser('list-layers', |
| help='List included layers', |
| description='Lists included layers') |
| parser_layers.add_argument('-o', '--output', help='Output file - defaults to stdout if not specified', |
| default=sys.stdout, type=argparse.FileType('w')) |
| parser_layers.set_defaults(func=list_layers) |
| |
| # dump default configuration file |
| parser_dconfig = subparsers.add_parser('dump-config', |
| help='Dump default config', |
| description='Dump default config to default_config.json') |
| parser_dconfig.set_defaults(func=dump_config) |
| |
| # export recipe info for packages in manifest |
| parser_export = subparsers.add_parser('manifest-info', |
| help='Export recipe info for a manifest', |
| description='Export recipe information using the manifest') |
| parser_export.add_argument('-c', '--config', help='load config from json file') |
| parser_export.add_argument('-o', '--output', help='Output file (tarball) - defaults to manifest name if not specified') |
| parser_export.add_argument('manifest', help='Manifest file') |
| parser_export.set_defaults(func=export_manifest_info) |
| |
| args = parser.parse_args() |
| |
| if args.debug: |
| logger.setLevel(logging.DEBUG) |
| logger.debug("Debug Enabled") |
| elif args.quiet: |
| logger.setLevel(logging.ERROR) |
| |
| ret = args.func(args) |
| |
| return ret |
| |
| |
| if __name__ == "__main__": |
| try: |
| ret = main() |
| except Exception: |
| ret = 1 |
| import traceback |
| traceback.print_exc() |
| sys.exit(ret) |