blob: 48277ec28ad8f0090f3fc82789a67f408a93970d [file] [log] [blame]
#!/usr/bin/env python3
#
# Copyright OpenEmbedded Contributors
#
# SPDX-License-Identifier: GPL-2.0-only
#
# Determine dependencies of python scripts or available python modules in a search path.
#
# Given the -d argument and a filename/filenames, returns the modules imported by those files.
# Given the -d argument and a directory/directories, recurses to find all
# python packages and modules, returns the modules imported by these.
# Given the -p argument and a path or paths, scans that path for available python modules/packages.
import argparse
import ast
import importlib
from importlib import machinery
import logging
import os.path
import sys
logger = logging.getLogger('pythondeps')
suffixes = importlib.machinery.all_suffixes()
class PythonDepError(Exception):
pass
class DependError(PythonDepError):
def __init__(self, path, error):
self.path = path
self.error = error
PythonDepError.__init__(self, error)
def __str__(self):
return "Failure determining dependencies of {}: {}".format(self.path, self.error)
class ImportVisitor(ast.NodeVisitor):
def __init__(self):
self.imports = set()
self.importsfrom = []
def visit_Import(self, node):
for alias in node.names:
self.imports.add(alias.name)
def visit_ImportFrom(self, node):
self.importsfrom.append((node.module, [a.name for a in node.names], node.level))
def walk_up(path):
while path:
yield path
path, _, _ = path.rpartition(os.sep)
def get_provides(path):
path = os.path.realpath(path)
def get_fn_name(fn):
for suffix in suffixes:
if fn.endswith(suffix):
return fn[:-len(suffix)]
isdir = os.path.isdir(path)
if isdir:
pkg_path = path
walk_path = path
else:
pkg_path = get_fn_name(path)
if pkg_path is None:
return
walk_path = os.path.dirname(path)
for curpath in walk_up(walk_path):
if not os.path.exists(os.path.join(curpath, '__init__.py')):
libdir = curpath
break
else:
libdir = ''
package_relpath = pkg_path[len(libdir)+1:]
package = '.'.join(package_relpath.split(os.sep))
if not isdir:
yield package, path
else:
if os.path.exists(os.path.join(path, '__init__.py')):
yield package, path
for dirpath, dirnames, filenames in os.walk(path):
relpath = dirpath[len(path)+1:]
if relpath:
if '__init__.py' not in filenames:
dirnames[:] = []
continue
else:
context = '.'.join(relpath.split(os.sep))
if package:
context = package + '.' + context
yield context, dirpath
else:
context = package
for fn in filenames:
adjusted_fn = get_fn_name(fn)
if not adjusted_fn or adjusted_fn == '__init__':
continue
fullfn = os.path.join(dirpath, fn)
if context:
yield context + '.' + adjusted_fn, fullfn
else:
yield adjusted_fn, fullfn
def get_code_depends(code_string, path=None, provide=None, ispkg=False):
try:
code = ast.parse(code_string, path)
except TypeError as exc:
raise DependError(path, exc)
except SyntaxError as exc:
raise DependError(path, exc)
visitor = ImportVisitor()
visitor.visit(code)
for builtin_module in sys.builtin_module_names:
if builtin_module in visitor.imports:
visitor.imports.remove(builtin_module)
if provide:
provide_elements = provide.split('.')
if ispkg:
provide_elements.append("__self__")
context = '.'.join(provide_elements[:-1])
package_path = os.path.dirname(path)
else:
context = None
package_path = None
levelzero_importsfrom = (module for module, names, level in visitor.importsfrom
if level == 0)
for module in visitor.imports | set(levelzero_importsfrom):
if context and path:
module_basepath = os.path.join(package_path, module.replace('.', '/'))
if os.path.exists(module_basepath):
# Implicit relative import
yield context + '.' + module, path
continue
for suffix in suffixes:
if os.path.exists(module_basepath + suffix):
# Implicit relative import
yield context + '.' + module, path
break
else:
yield module, path
else:
yield module, path
for module, names, level in visitor.importsfrom:
if level == 0:
continue
elif not provide:
raise DependError("Error: ImportFrom non-zero level outside of a package: {0}".format((module, names, level)), path)
elif level > len(provide_elements):
raise DependError("Error: ImportFrom level exceeds package depth: {0}".format((module, names, level)), path)
else:
context = '.'.join(provide_elements[:-level])
if module:
if context:
yield context + '.' + module, path
else:
yield module, path
def get_file_depends(path):
try:
code_string = open(path, 'r').read()
except (OSError, IOError) as exc:
raise DependError(path, exc)
return get_code_depends(code_string, path)
def get_depends_recursive(directory):
directory = os.path.realpath(directory)
provides = dict((v, k) for k, v in get_provides(directory))
for filename, provide in provides.items():
if os.path.isdir(filename):
filename = os.path.join(filename, '__init__.py')
ispkg = True
elif not filename.endswith('.py'):
continue
else:
ispkg = False
with open(filename, 'r') as f:
source = f.read()
depends = get_code_depends(source, filename, provide, ispkg)
for depend, by in depends:
yield depend, by
def get_depends(path):
if os.path.isdir(path):
return get_depends_recursive(path)
else:
return get_file_depends(path)
def main():
logging.basicConfig()
parser = argparse.ArgumentParser(description='Determine dependencies and provided packages for python scripts/modules')
parser.add_argument('path', nargs='+', help='full path to content to be processed')
group = parser.add_mutually_exclusive_group()
group.add_argument('-p', '--provides', action='store_true',
help='given a path, display the provided python modules')
group.add_argument('-d', '--depends', action='store_true',
help='given a filename, display the imported python modules')
args = parser.parse_args()
if args.provides:
modules = set()
for path in args.path:
for provide, fn in get_provides(path):
modules.add(provide)
for module in sorted(modules):
print(module)
elif args.depends:
for path in args.path:
try:
modules = get_depends(path)
except PythonDepError as exc:
logger.error(str(exc))
sys.exit(1)
for module, imp_by in modules:
print("{}\t{}".format(module, imp_by))
else:
parser.print_help()
sys.exit(2)
if __name__ == '__main__':
main()