blob: 4d65a99258d85719977a7b069f882f6ec609e6bc [file] [log] [blame]
#!/usr/bin/env python3
# Script to extract information from image manifests
#
# Copyright (C) 2018 Intel Corporation
# Copyright (C) 2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: GPL-2.0-only
#
import sys
import os
import argparse
import logging
import json
import shutil
import tempfile
import tarfile
from collections import OrderedDict
scripts_path = os.path.dirname(__file__)
lib_path = scripts_path + '/../lib'
sys.path = sys.path + [lib_path]
import scriptutils
logger = scriptutils.logger_create(os.path.basename(__file__))
import argparse_oe
import scriptpath
bitbakepath = scriptpath.add_bitbake_lib_path()
if not bitbakepath:
logger.error("Unable to find bitbake by searching parent directory of this script or PATH")
sys.exit(1)
logger.debug('Using standard bitbake path %s' % bitbakepath)
scriptpath.add_oe_lib_path()
import bb.tinfoil
import bb.utils
import oe.utils
import oe.recipeutils
def get_pkg_list(manifest):
pkglist = []
with open(manifest, 'r') as f:
for line in f:
linesplit = line.split()
if len(linesplit) == 3:
# manifest file
pkglist.append(linesplit[0])
elif len(linesplit) == 1:
# build dependency file
pkglist.append(linesplit[0])
return sorted(pkglist)
def list_packages(args):
pkglist = get_pkg_list(args.manifest)
for pkg in pkglist:
print('%s' % pkg)
def pkg2recipe(tinfoil, pkg):
if "-native" in pkg:
logger.info('skipping %s' % pkg)
return None
pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR')
pkgdatafile = os.path.join(pkgdata_dir, 'runtime-reverse', pkg)
logger.debug('pkgdatafile %s' % pkgdatafile)
try:
f = open(pkgdatafile, 'r')
for line in f:
if line.startswith('PN:'):
recipe = line.split(':', 1)[1].strip()
return recipe
except Exception:
logger.warning('%s is missing' % pkgdatafile)
return None
def get_recipe_list(manifest, tinfoil):
pkglist = get_pkg_list(manifest)
recipelist = []
for pkg in pkglist:
recipe = pkg2recipe(tinfoil,pkg)
if recipe:
if not recipe in recipelist:
recipelist.append(recipe)
return sorted(recipelist)
def list_recipes(args):
import bb.tinfoil
with bb.tinfoil.Tinfoil() as tinfoil:
tinfoil.logger.setLevel(logger.getEffectiveLevel())
tinfoil.prepare(config_only=True)
recipelist = get_recipe_list(args.manifest, tinfoil)
for recipe in sorted(recipelist):
print('%s' % recipe)
def list_layers(args):
def find_git_repo(pth):
checkpth = pth
while checkpth != os.sep:
if os.path.exists(os.path.join(checkpth, '.git')):
return checkpth
checkpth = os.path.dirname(checkpth)
return None
def get_git_remote_branch(repodir):
try:
stdout, _ = bb.process.run(['git', 'rev-parse', '--abbrev-ref', '--symbolic-full-name', '@{u}'], cwd=repodir)
except bb.process.ExecutionError as e:
stdout = None
if stdout:
return stdout.strip()
else:
return None
def get_git_head_commit(repodir):
try:
stdout, _ = bb.process.run(['git', 'rev-parse', 'HEAD'], cwd=repodir)
except bb.process.ExecutionError as e:
stdout = None
if stdout:
return stdout.strip()
else:
return None
def get_git_repo_url(repodir, remote='origin'):
import bb.process
# Try to get upstream repo location from origin remote
try:
stdout, _ = bb.process.run(['git', 'remote', '-v'], cwd=repodir)
except bb.process.ExecutionError as e:
stdout = None
if stdout:
for line in stdout.splitlines():
splitline = line.split()
if len(splitline) > 1:
if splitline[0] == remote and scriptutils.is_src_url(splitline[1]):
return splitline[1]
return None
with bb.tinfoil.Tinfoil() as tinfoil:
tinfoil.logger.setLevel(logger.getEffectiveLevel())
tinfoil.prepare(config_only=False)
layers = OrderedDict()
for layerdir in tinfoil.config_data.getVar('BBLAYERS').split():
layerdata = OrderedDict()
layername = os.path.basename(layerdir)
logger.debug('layername %s, layerdir %s' % (layername, layerdir))
if layername in layers:
logger.warning('layername %s is not unique in configuration' % layername)
layername = os.path.basename(os.path.dirname(layerdir)) + '_' + os.path.basename(layerdir)
logger.debug('trying layername %s' % layername)
if layername in layers:
logger.error('Layer name %s is not unique in configuration' % layername)
sys.exit(2)
repodir = find_git_repo(layerdir)
if repodir:
remotebranch = get_git_remote_branch(repodir)
remote = 'origin'
if remotebranch and '/' in remotebranch:
rbsplit = remotebranch.split('/', 1)
layerdata['actual_branch'] = rbsplit[1]
remote = rbsplit[0]
layerdata['vcs_url'] = get_git_repo_url(repodir, remote)
if os.path.abspath(repodir) != os.path.abspath(layerdir):
layerdata['vcs_subdir'] = os.path.relpath(layerdir, repodir)
commit = get_git_head_commit(repodir)
if commit:
layerdata['vcs_commit'] = commit
layers[layername] = layerdata
json.dump(layers, args.output, indent=2)
def get_recipe(args):
with bb.tinfoil.Tinfoil() as tinfoil:
tinfoil.logger.setLevel(logger.getEffectiveLevel())
tinfoil.prepare(config_only=True)
recipe = pkg2recipe(tinfoil, args.package)
print(' %s package provided by %s' % (args.package, recipe))
def pkg_dependencies(args):
def get_recipe_info(tinfoil, recipe):
try:
info = tinfoil.get_recipe_info(recipe)
except Exception:
logger.error('Failed to get recipe info for: %s' % recipe)
sys.exit(1)
if not info:
logger.warning('No recipe info found for: %s' % recipe)
sys.exit(1)
append_files = tinfoil.get_file_appends(info.fn)
appends = True
data = tinfoil.parse_recipe_file(info.fn, appends, append_files)
data.pn = info.pn
data.pv = info.pv
return data
def find_dependencies(tinfoil, assume_provided, recipe_info, packages, rn, order):
spaces = ' ' * order
data = recipe_info[rn]
if args.native:
logger.debug('%s- %s' % (spaces, data.pn))
elif "-native" not in data.pn:
if "cross" not in data.pn:
logger.debug('%s- %s' % (spaces, data.pn))
depends = []
for dep in data.depends:
if dep not in assume_provided:
depends.append(dep)
# First find all dependencies not in package list.
for dep in depends:
if dep not in packages:
packages.append(dep)
dep_data = get_recipe_info(tinfoil, dep)
# Do this once now to reduce the number of bitbake calls.
dep_data.depends = dep_data.getVar('DEPENDS').split()
recipe_info[dep] = dep_data
# Then recursively analyze all of the dependencies for the current recipe.
for dep in depends:
find_dependencies(tinfoil, assume_provided, recipe_info, packages, dep, order + 1)
with bb.tinfoil.Tinfoil() as tinfoil:
tinfoil.logger.setLevel(logger.getEffectiveLevel())
tinfoil.prepare()
assume_provided = tinfoil.config_data.getVar('ASSUME_PROVIDED').split()
logger.debug('assumed provided:')
for ap in sorted(assume_provided):
logger.debug(' - %s' % ap)
recipe = pkg2recipe(tinfoil, args.package)
data = get_recipe_info(tinfoil, recipe)
data.depends = []
depends = data.getVar('DEPENDS').split()
for dep in depends:
if dep not in assume_provided:
data.depends.append(dep)
recipe_info = dict([(recipe, data)])
packages = []
find_dependencies(tinfoil, assume_provided, recipe_info, packages, recipe, order=1)
print('\nThe following packages are required to build %s' % recipe)
for p in sorted(packages):
data = recipe_info[p]
if "-native" not in data.pn:
if "cross" not in data.pn:
print(" %s (%s)" % (data.pn,p))
if args.native:
print('\nThe following native packages are required to build %s' % recipe)
for p in sorted(packages):
data = recipe_info[p]
if "-native" in data.pn:
print(" %s(%s)" % (data.pn,p))
if "cross" in data.pn:
print(" %s(%s)" % (data.pn,p))
def default_config():
vlist = OrderedDict()
vlist['PV'] = 'yes'
vlist['SUMMARY'] = 'no'
vlist['DESCRIPTION'] = 'no'
vlist['SECTION'] = 'no'
vlist['LICENSE'] = 'yes'
vlist['HOMEPAGE'] = 'no'
vlist['BUGTRACKER'] = 'no'
vlist['PROVIDES'] = 'no'
vlist['BBCLASSEXTEND'] = 'no'
vlist['DEPENDS'] = 'no'
vlist['PACKAGECONFIG'] = 'no'
vlist['SRC_URI'] = 'yes'
vlist['SRCREV'] = 'yes'
vlist['EXTRA_OECONF'] = 'no'
vlist['EXTRA_OESCONS'] = 'no'
vlist['EXTRA_OECMAKE'] = 'no'
vlist['EXTRA_OEMESON'] = 'no'
clist = OrderedDict()
clist['variables'] = vlist
clist['filepath'] = 'no'
clist['sha256sum'] = 'no'
clist['layerdir'] = 'no'
clist['layer'] = 'no'
clist['inherits'] = 'no'
clist['source_urls'] = 'no'
clist['packageconfig_opts'] = 'no'
clist['patches'] = 'no'
clist['packagedir'] = 'no'
return clist
def dump_config(args):
config = default_config()
f = open('default_config.json', 'w')
json.dump(config, f, indent=2)
logger.info('Default config list dumped to default_config.json')
def export_manifest_info(args):
def handle_value(value):
if value:
return oe.utils.squashspaces(value)
else:
return value
if args.config:
logger.debug('config: %s' % args.config)
f = open(args.config, 'r')
config = json.load(f, object_pairs_hook=OrderedDict)
else:
config = default_config()
if logger.isEnabledFor(logging.DEBUG):
print('Configuration:')
json.dump(config, sys.stdout, indent=2)
print('')
tmpoutdir = tempfile.mkdtemp(prefix=os.path.basename(__file__)+'-')
logger.debug('tmp dir: %s' % tmpoutdir)
# export manifest
shutil.copy2(args.manifest,os.path.join(tmpoutdir, "manifest"))
with bb.tinfoil.Tinfoil(tracking=True) as tinfoil:
tinfoil.logger.setLevel(logger.getEffectiveLevel())
tinfoil.prepare(config_only=False)
pkglist = get_pkg_list(args.manifest)
# export pkg list
f = open(os.path.join(tmpoutdir, "pkgs"), 'w')
for pkg in pkglist:
f.write('%s\n' % pkg)
f.close()
recipelist = []
for pkg in pkglist:
recipe = pkg2recipe(tinfoil,pkg)
if recipe:
if not recipe in recipelist:
recipelist.append(recipe)
recipelist.sort()
# export recipe list
f = open(os.path.join(tmpoutdir, "recipes"), 'w')
for recipe in recipelist:
f.write('%s\n' % recipe)
f.close()
try:
rvalues = OrderedDict()
for pn in sorted(recipelist):
logger.debug('Package: %s' % pn)
rd = tinfoil.parse_recipe(pn)
rvalues[pn] = OrderedDict()
for varname in config['variables']:
if config['variables'][varname] == 'yes':
rvalues[pn][varname] = handle_value(rd.getVar(varname))
fpth = rd.getVar('FILE')
layerdir = oe.recipeutils.find_layerdir(fpth)
if config['filepath'] == 'yes':
rvalues[pn]['filepath'] = os.path.relpath(fpth, layerdir)
if config['sha256sum'] == 'yes':
rvalues[pn]['sha256sum'] = bb.utils.sha256_file(fpth)
if config['layerdir'] == 'yes':
rvalues[pn]['layerdir'] = layerdir
if config['layer'] == 'yes':
rvalues[pn]['layer'] = os.path.basename(layerdir)
if config['inherits'] == 'yes':
gr = set(tinfoil.config_data.getVar("__inherit_cache") or [])
lr = set(rd.getVar("__inherit_cache") or [])
rvalues[pn]['inherits'] = sorted({os.path.splitext(os.path.basename(r))[0] for r in lr if r not in gr})
if config['source_urls'] == 'yes':
rvalues[pn]['source_urls'] = []
for url in (rd.getVar('SRC_URI') or '').split():
if not url.startswith('file://'):
url = url.split(';')[0]
rvalues[pn]['source_urls'].append(url)
if config['packageconfig_opts'] == 'yes':
rvalues[pn]['packageconfig_opts'] = OrderedDict()
for key in rd.getVarFlags('PACKAGECONFIG').keys():
if key == 'doc':
continue
rvalues[pn]['packageconfig_opts'][key] = rd.getVarFlag('PACKAGECONFIG', key)
if config['patches'] == 'yes':
patches = oe.recipeutils.get_recipe_patches(rd)
rvalues[pn]['patches'] = []
if patches:
recipeoutdir = os.path.join(tmpoutdir, pn, 'patches')
bb.utils.mkdirhier(recipeoutdir)
for patch in patches:
# Patches may be in other layers too
patchlayerdir = oe.recipeutils.find_layerdir(patch)
# patchlayerdir will be None for remote patches, which we ignore
# (since currently they are considered as part of sources)
if patchlayerdir:
rvalues[pn]['patches'].append((os.path.basename(patchlayerdir), os.path.relpath(patch, patchlayerdir)))
shutil.copy(patch, recipeoutdir)
if config['packagedir'] == 'yes':
pn_dir = os.path.join(tmpoutdir, pn)
bb.utils.mkdirhier(pn_dir)
f = open(os.path.join(pn_dir, 'recipe.json'), 'w')
json.dump(rvalues[pn], f, indent=2)
f.close()
with open(os.path.join(tmpoutdir, 'recipes.json'), 'w') as f:
json.dump(rvalues, f, indent=2)
if args.output:
outname = os.path.basename(args.output)
else:
outname = os.path.splitext(os.path.basename(args.manifest))[0]
if outname.endswith('.tar.gz'):
outname = outname[:-7]
elif outname.endswith('.tgz'):
outname = outname[:-4]
tarfn = outname
if tarfn.endswith(os.sep):
tarfn = tarfn[:-1]
if not tarfn.endswith(('.tar.gz', '.tgz')):
tarfn += '.tar.gz'
with open(tarfn, 'wb') as f:
with tarfile.open(None, "w:gz", f) as tar:
tar.add(tmpoutdir, outname)
finally:
shutil.rmtree(tmpoutdir)
def main():
parser = argparse_oe.ArgumentParser(description="Image manifest utility",
epilog="Use %(prog)s <subcommand> --help to get help on a specific command")
parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true')
parser.add_argument('-q', '--quiet', help='Print only errors', action='store_true')
subparsers = parser.add_subparsers(dest="subparser_name", title='subcommands', metavar='<subcommand>')
subparsers.required = True
# get recipe info
parser_get_recipes = subparsers.add_parser('recipe-info',
help='Get recipe info',
description='Get recipe information for a package')
parser_get_recipes.add_argument('package', help='Package name')
parser_get_recipes.set_defaults(func=get_recipe)
# list runtime dependencies
parser_pkg_dep = subparsers.add_parser('list-depends',
help='List dependencies',
description='List dependencies required to build the package')
parser_pkg_dep.add_argument('--native', help='also print native and cross packages', action='store_true')
parser_pkg_dep.add_argument('package', help='Package name')
parser_pkg_dep.set_defaults(func=pkg_dependencies)
# list recipes
parser_recipes = subparsers.add_parser('list-recipes',
help='List recipes producing packages within an image',
description='Lists recipes producing the packages that went into an image, using the manifest and pkgdata')
parser_recipes.add_argument('manifest', help='Manifest file')
parser_recipes.set_defaults(func=list_recipes)
# list packages
parser_packages = subparsers.add_parser('list-packages',
help='List packages within an image',
description='Lists packages that went into an image, using the manifest')
parser_packages.add_argument('manifest', help='Manifest file')
parser_packages.set_defaults(func=list_packages)
# list layers
parser_layers = subparsers.add_parser('list-layers',
help='List included layers',
description='Lists included layers')
parser_layers.add_argument('-o', '--output', help='Output file - defaults to stdout if not specified',
default=sys.stdout, type=argparse.FileType('w'))
parser_layers.set_defaults(func=list_layers)
# dump default configuration file
parser_dconfig = subparsers.add_parser('dump-config',
help='Dump default config',
description='Dump default config to default_config.json')
parser_dconfig.set_defaults(func=dump_config)
# export recipe info for packages in manifest
parser_export = subparsers.add_parser('manifest-info',
help='Export recipe info for a manifest',
description='Export recipe information using the manifest')
parser_export.add_argument('-c', '--config', help='load config from json file')
parser_export.add_argument('-o', '--output', help='Output file (tarball) - defaults to manifest name if not specified')
parser_export.add_argument('manifest', help='Manifest file')
parser_export.set_defaults(func=export_manifest_info)
args = parser.parse_args()
if args.debug:
logger.setLevel(logging.DEBUG)
logger.debug("Debug Enabled")
elif args.quiet:
logger.setLevel(logging.ERROR)
ret = args.func(args)
return ret
if __name__ == "__main__":
try:
ret = main()
except Exception:
ret = 1
import traceback
traceback.print_exc()
sys.exit(ret)