blob: 1af1443f476b3bb15d95af6320aa4927f550d522 [file] [log] [blame]
# This script is used as a bitbake task to create a new python manifest
# $ bitbake python -c create_manifest
#
# Our goal is to keep python-core as small as posible and add other python
# packages only when the user needs them, hence why we split upstream python
# into several packages.
#
# In a very simplistic way what this does is:
# Launch python and see specifically what is required for it to run at a minimum
#
# Go through the python-manifest file and launch a separate task for every single
# one of the files on each package, this task will check what was required for that
# specific module to run, these modules will be called dependencies.
# The output of such task will be a list of the modules or dependencies that were
# found for that file.
#
# Such output will be parsed by this script, we will look for each dependency on the
# manifest and if we find that another package already includes it, then we will add
# that package as an RDEPENDS to the package we are currently checking; in case we dont
# find the current dependency on any other package we will add it to the current package
# as part of FILES.
#
#
# This way we will create a new manifest from the data structure that was built during
# this process, ont this new manifest each package will contain specifically only
# what it needs to run.
#
# There are some caveats which we try to deal with, such as repeated files on different
# packages, packages that include folders, wildcards, and special packages.
# Its also important to note that this method only works for python files, and shared
# libraries. Static libraries, header files and binaries need to be dealt with manually.
#
# Author: Alejandro Enedino Hernandez Samaniego "aehs29" <aehs29@gmail.com>
import sys
import subprocess
import json
import os
import collections
# Hack to get native python search path (for folders), not fond of it but it works for now
pivot='recipe-sysroot-native'
for p in sys.path:
if pivot in p:
nativelibfolder=p[:p.find(pivot)+len(pivot)]
# Empty dict to hold the whole manifest
new_manifest = collections.OrderedDict()
# Check for repeated files, folders and wildcards
allfiles=[]
repeated=[]
wildcards=[]
hasfolders=[]
allfolders=[]
def isFolder(value):
if os.path.isdir(value.replace('${libdir}',nativelibfolder+'/usr/lib')) or os.path.isdir(value.replace('${libdir}',nativelibfolder+'/usr/lib64')) or os.path.isdir(value.replace('${libdir}',nativelibfolder+'/usr/lib32')):
return True
else:
return False
# Read existing JSON manifest
with open('python2-manifest.json') as manifest:
old_manifest = json.load(manifest, object_pairs_hook=collections.OrderedDict)
# First pass to get core-package functionality, because we base everything on the fact that core is actually working
# Not exactly the same so it should not be a function
print ("Getting dependencies for core package:")
# Special call to check for core package
output = subprocess.check_output([sys.executable, 'get_module_deps2.py', 'python-core-package'])
for item in output.split():
# We append it so it doesnt hurt what we currently have:
if item not in old_manifest['core']['files']:
# We use the same data structure since its the one which will be used to check
# dependencies for other packages
old_manifest['core']['files'].append(item)
for value in old_manifest['core']['files']:
# Ignore folders, since we don't import those, difficult to handle multilib
if isFolder(value):
# Pass it directly
if value not in old_manifest['core']['files']:
old_manifest['core']['files'].append(value)
# Ignore binaries, since we don't import those, assume it was added correctly (manually)
if '${bindir}' in value:
# Pass it directly
if value not in old_manifest['core']['files']:
old_manifest['core']['files'].append(value)
continue
# Ignore empty values
if value == '':
continue
if '${includedir}' in value:
if value not in old_manifest['core']['files']:
old_manifest['core']['files'].append(value)
continue
# Get module name , shouldnt be affected by libdir/bindir
value = os.path.splitext(os.path.basename(os.path.normpath(value)))[0]
# Launch separate task for each module for deterministic behavior
# Each module will only import what is necessary for it to work in specific
print ('Getting dependencies for module: %s' % value)
output = subprocess.check_output([sys.executable, 'get_module_deps2.py', '%s' % value])
for item in output.split():
# We append it so it doesnt hurt what we currently have:
if item not in old_manifest['core']['files']:
old_manifest['core']['files'].append(item)
# We check which packages include folders
for key in old_manifest:
for value in old_manifest[key]['files']:
# Ignore folders, since we don't import those, difficult to handle multilib
if isFolder(value):
print ('%s is a folder' % value)
if key not in hasfolders:
hasfolders.append(key)
if value not in allfolders:
allfolders.append(value)
for key in old_manifest:
# Use an empty dict as data structure to hold data for each package and fill it up
new_manifest[key] = collections.OrderedDict()
new_manifest[key]['summary'] = old_manifest[key]['summary']
new_manifest[key]['rdepends']=[]
new_manifest[key]['files'] = []
# All packages should depend on core
if key != 'core':
new_manifest[key]['rdepends'].append('core')
# Handle special cases, we assume that when they were manually added
# to the manifest we knew what we were doing.
print ('Handling package %s' % key)
special_packages=['misc', 'modules', 'dev']
if key in special_packages or 'staticdev' in key:
print('Passing %s package directly' % key)
new_manifest[key]=old_manifest[key]
continue
for value in old_manifest[key]['files']:
# We already handled core on the first pass
if key == 'core':
new_manifest[key]['files'].append(value)
continue
# Ignore folders, since we don't import those, difficult to handle multilib
if isFolder(value):
# Pass folders directly
new_manifest[key]['files'].append(value)
# Ignore binaries, since we don't import those
if '${bindir}' in value:
# Pass it directly to the new manifest data structure
if value not in new_manifest[key]['files']:
new_manifest[key]['files'].append(value)
continue
# Ignore empty values
if value == '':
continue
if '${includedir}' in value:
if value not in new_manifest[key]['files']:
new_manifest[key]['files'].append(value)
continue
# Get module name , shouldnt be affected by libdir/bindir
value = os.path.splitext(os.path.basename(os.path.normpath(value)))[0]
# Launch separate task for each module for deterministic behavior
# Each module will only import what is necessary for it to work in specific
print ('Getting dependencies for module: %s' % value)
output = subprocess.check_output([sys.executable, 'get_module_deps2.py', '%s' % value])
# We can print dependencies for debugging purposes
#print (output)
# Output will have all dependencies
for item in output.split():
# Warning: This first part is ugly
# One of the dependencies that was found, could be inside of one of the folders included by another package
# We need to check if this happens so we can add the package containing the folder as an RDEPENDS
# e.g. Folder encodings contained in codecs
# This would be solved if no packages included any folders
# This can be done in two ways:
# 1 - We assume that if we take out the filename from the path we would get
# the folder string, then we would check if folder string is in the list of folders
# This would not work if a package contains a folder which contains another folder
# e.g. path/folder1/folder2/filename folder_string= path/folder1/folder2
# folder_string would not match any value contained in the list of folders
#
# 2 - We do it the other way around, checking if the folder is contained in the path
# e.g. path/folder1/folder2/filename folder_string= path/folder1/folder2
# is folder_string inside path/folder1/folder2/filename?,
# Yes, it works, but we waste a couple of milliseconds.
inFolders=False
for folder in allfolders:
if folder in item:
inFolders = True # Did we find a folder?
folderFound = False # Second flag to break inner for
# Loop only through packages which contain folders
for keyfolder in hasfolders:
if (folderFound == False):
#print("Checking folder %s on package %s" % (item,keyfolder))
for file_folder in old_manifest[keyfolder]['files']:
if file_folder==folder:
print ('%s found in %s' % (folder, keyfolder))
folderFound = True
if keyfolder not in new_manifest[key]['rdepends'] and keyfolder != key:
new_manifest[key]['rdepends'].append(keyfolder)
else:
break
# A folder was found so we're done with this item, we can go on
if inFolders:
continue
# We might already have it on the dictionary since it could depend on a (previously checked) module
if item not in new_manifest[key]['files']:
# Handle core as a special package, we already did it so we pass it to NEW data structure directly
if key=='core':
print('Adding %s to %s FILES' % (item, key))
if item.endswith('*'):
wildcards.append(item)
new_manifest[key]['files'].append(item)
# Check for repeated files
if item not in allfiles:
allfiles.append(item)
else:
repeated.append(item)
else:
# Check if this dependency is already contained on another package, so we add it
# as an RDEPENDS, or if its not, it means it should be contained on the current
# package, so we should add it to FILES
for newkey in old_manifest:
# Debug
#print("Checking %s " % item + " in %s" % newkey)
if item in old_manifest[newkey]['files']:
# Since were nesting, we need to check its not the same key
if(newkey!=key):
if newkey not in new_manifest[key]['rdepends']:
# Add it to the new manifest data struct
# Debug
print('Adding %s to %s RDEPENDS, because it contains %s' % (newkey, key, item))
new_manifest[key]['rdepends'].append(newkey)
break
else:
# Debug
print('Adding %s to %s FILES' % (item, key))
# Since it wasnt found on another package, its not an RDEP, so add it to FILES for this package
new_manifest[key]['files'].append(item)
if item.endswith('*'):
wildcards.append(item)
if item not in allfiles:
allfiles.append(item)
else:
repeated.append(item)
print ('The following files are repeated (contained in more than one package), please check which package should get it:')
print (repeated)
print('The following files contain wildcards, please check they are necessary')
print(wildcards)
print('The following files contain folders, please check they are necessary')
print(hasfolders)
# Sort it just so it looks nice
for key in new_manifest:
new_manifest[key]['files'].sort()
new_manifest[key]['rdepends'].sort()
# Create the manifest from the data structure that was built
with open('python2-manifest.json.new','w') as outfile:
json.dump(new_manifest,outfile, indent=4)