poky/meta/recipes-devtools/python/python3/create_manifest3.py

# This script is used as a bitbake task to create a new python manifest
# $ bitbake python -c create_manifest
#
# Our goal is to keep python-core as small as posible and add other python
# packages only when the user needs them, hence why we split upstream python
# into several packages.
#
# In a very simplistic way what this does is: 
# Launch python and see specifically what is required for it to run at a minimum
#
# Go through the python-manifest file and launch a separate task for every single
# one of the files on each package, this task will check what was required for that
# specific module to run, these modules will be called dependencies.
# The output of such task will be a list of the modules or dependencies that were
# found for that file.
#
# Such output will be parsed by this script, we will look for each dependency on the
# manifest and if we find that another package already includes it, then we will add
# that package as an RDEPENDS to the package we are currently checking; in case we dont
# find the current dependency on any other package we will add it to the current package
# as part of FILES.
#
#
# This way we will create a new manifest from the data structure that was built during
# this process, ont this new manifest each package will contain specifically only
# what it needs to run.
#
# There are some caveats which we try to deal with, such as repeated files on different
# packages, packages that include folders, wildcards, and special packages.
# Its also important to note that this method only works for python files, and shared
# libraries. Static libraries, header files and binaries need to be dealt with manually.
#
# This script differs from its python2 version mostly on how shared libraries are handled
# The manifest file for python3 has an extra field which contains the cached files for
# each package.
# Tha method to handle cached files does not work when a module includes a folder which
# itself contains the pycache folder, gladly this is almost never the case.
#
# Author: Alejandro Enedino Hernandez Samaniego "aehs29" <aehs29@gmail.com>


import sys
import subprocess
import json
import os

# Hack to get native python search path (for folders), not fond of it but it works for now
pivot='recipe-sysroot-native'
for p in sys.path:
  if pivot in p:
    nativelibfolder=p[:p.find(pivot)+len(pivot)]

# Empty dict to hold the whole manifest
new_manifest = {}

# Check for repeated files, folders and wildcards
allfiles=[]
repeated=[]
wildcards=[]

hasfolders=[]
allfolders=[]

def isFolder(value):
  if os.path.isdir(value.replace('${libdir}',nativelibfolder+'/usr/lib')) or os.path.isdir(value.replace('${libdir}',nativelibfolder+'/usr/lib64')) or os.path.isdir(value.replace('${libdir}',nativelibfolder+'/usr/lib32')):
    return True
  else:
    return False

def isCached(item):
  if '__pycache__' in item:
    return True
  else:
    return False

# Read existing JSON manifest
with open('python3-manifest.json') as manifest:
  old_manifest=json.load(manifest)


# First pass to get core-package functionality, because we base everything on the fact that core is actually working
# Not exactly the same so it should not be a function
print ('Getting dependencies for package: core')

# Special call to check for core package
output = subprocess.check_output([sys.executable, 'get_module_deps3.py', 'python-core-package']).decode('utf8')
for item in output.split():
    # We append it so it doesnt hurt what we currently have:
    if isCached(item):
        if item not in old_manifest['core']['cached']:
            # We use the same data structure since its the one which will be used to check
            # dependencies for other packages
            old_manifest['core']['cached'].append(item)
    else:
        if item not in old_manifest['core']['files']:
            # We use the same data structure since its the one which will be used to check
            # dependencies for other packages
            old_manifest['core']['files'].append(item)

for value in old_manifest['core']['files']:
  # Ignore folders, since we don't import those, difficult to handle multilib
  if isFolder(value):
    # Pass it directly
    if isCached(value):
        if value not in old_manifest['core']['cached']:
            old_manifest['core']['cached'].append(value)
    else:
        if value not in old_manifest['core']['files']:
            old_manifest['core']['files'].append(value)
    continue
  # Ignore binaries, since we don't import those, assume it was added correctly (manually)
  if '${bindir}' in value:
    # Pass it directly
    if value not in old_manifest['core']['files']:
      old_manifest['core']['files'].append(value)
    continue
  # Ignore empty values
  if value == '':
    continue
  if '${includedir}' in value:
    if value not in old_manifest['core']['files']:
      old_manifest['core']['files'].append(value)
    continue
  # Get module name , shouldnt be affected by libdir/bindir
  value = os.path.splitext(os.path.basename(os.path.normpath(value)))[0]

  # Launch separate task for each module for deterministic behavior
  # Each module will only import what is necessary for it to work in specific
  print ('Getting dependencies for module: %s' % value)
  output = subprocess.check_output([sys.executable, 'get_module_deps3.py', '%s' % value]).decode('utf8')
  print ('The following dependencies were found for module %s:\n' % value)
  print (output)
  for item in output.split():
    # We append it so it doesnt hurt what we currently have:
    if isCached(item):
        if item not in old_manifest['core']['cached']:
            # We use the same data structure since its the one which will be used to check
            # dependencies for other packages
            old_manifest['core']['cached'].append(item)
    else:
        if item not in old_manifest['core']['files']:
            # We use the same data structure since its the one which will be used to check
            # dependencies for other packages
            old_manifest['core']['files'].append(item)


# We check which packages include folders
for key in old_manifest:
    for value in old_manifest[key]['files']:
        # Ignore folders, since we don't import those, difficult to handle multilib
        if isFolder(value):
            print ('%s is a folder' % value)
            if key not in hasfolders:
                hasfolders.append(key)
            if value not in allfolders:
                allfolders.append(value)

for key in old_manifest:
    # Use an empty dict as data structure to hold data for each package and fill it up
    new_manifest[key]={}
    new_manifest[key]['files']=[]
    
    new_manifest[key]['rdepends']=[]
    # All packages should depend on core
    if key != 'core':
         new_manifest[key]['rdepends'].append('core')
         new_manifest[key]['cached']=[]
    else:
         new_manifest[key]['cached']=old_manifest[key]['cached']
    new_manifest[key]['summary']=old_manifest[key]['summary']

    # Handle special cases, we assume that when they were manually added 
    # to the manifest we knew what we were doing.
    print('\n')
    print('--------------------------')
    print ('Handling package %s' % key)
    print('--------------------------')
    special_packages=['misc', 'modules', 'dev']
    if key in special_packages or 'staticdev' in key:
        print('Passing %s package directly' % key)
        new_manifest[key]=old_manifest[key]
        continue

    for value in old_manifest[key]['files']:
        # We already handled core on the first pass
        if key == 'core':
            new_manifest[key]['files'].append(value)
            continue
        # Ignore folders, since we don't import those, difficult to handle multilib
        if isFolder(value):
            # Pass folders directly
            new_manifest[key]['files'].append(value)
        # Ignore binaries, since we don't import those
        if '${bindir}' in value:
            # Pass it directly to the new manifest data structure
            if value not in new_manifest[key]['files']:
                new_manifest[key]['files'].append(value)
            continue
        # Ignore empty values
        if value == '':
            continue
        if '${includedir}' in value:
            if value not in new_manifest[key]['files']:
                new_manifest[key]['files'].append(value)
            continue

        # Get module name , shouldnt be affected by libdir/bindir
        # We need to check if the imported module comes from another (e.g. sqlite3.dump)
        path,value = os.path.split(value)
        path = os.path.basename(path)
        value = os.path.splitext(os.path.basename(value))[0]

        # If this condition is met, it means we need to import it from another module
        # or its the folder itself (e.g. unittest)
        if path == key:
          if value:
            value = path + '.' + value
          else:
            value = path

        # Launch separate task for each module for deterministic behavior
        # Each module will only import what is necessary for it to work in specific
        print ('\nGetting dependencies for module: %s' % value)
        output = subprocess.check_output([sys.executable, 'get_module_deps3.py', '%s' % value]).decode('utf8')
        # We can print dependencies for debugging purposes
        print ('The following dependencies were found for module %s:\n' % value)
        print (output)
        # Output will have all dependencies

        reportFILES = []
        reportRDEPS = []

        for item in output.split():

            # Warning: This first part is ugly
            # One of the dependencies that was found, could be inside of one of the folders included by another package
            # We need to check if this happens so we can add the package containing the folder as an rdependency
            # e.g. Folder encodings contained in codecs
            # This would be solved if no packages included any folders

            # This can be done in two ways:
            # 1 - We assume that if we take out the filename from the path we would get
            #   the folder string, then we would check if folder string is in the list of folders
            #   This would not work if a package contains a folder which contains another folder
            #   e.g. path/folder1/folder2/filename  folder_string= path/folder1/folder2
            #   folder_string would not match any value contained in the list of folders
            #
            # 2 - We do it the other way around, checking if the folder is contained in the path
            #   e.g. path/folder1/folder2/filename  folder_string= path/folder1/folder2
            #   is folder_string inside path/folder1/folder2/filename?, 
            #   Yes, it works, but we waste a couple of milliseconds.

            inFolders=False
            for folder in allfolders:
                if folder in item:
                    inFolders = True # Did we find a folder?
                    folderFound = False # Second flag to break inner for
                    # Loop only through packages which contain folders
                    for keyfolder in hasfolders:
                        if (folderFound == False):
                            #print('Checking folder %s on package %s' % (item,keyfolder))
                            for file_folder in old_manifest[keyfolder]['files'] or file_folder in old_manifest[keyfolder]['cached']:
                                if file_folder==folder:
                                    print ('%s folder found in %s' % (folder, keyfolder))
                                    folderFound = True
                                    if keyfolder not in new_manifest[key]['rdepends'] and keyfolder != key:
                                        new_manifest[key]['rdepends'].append(keyfolder)
                                    
                        else:
                            break

            # A folder was found so we're done with this item, we can go on
            if inFolders:
                continue


            # We might already have it on the dictionary since it could depend on a (previously checked) module
            if item not in new_manifest[key]['files'] and item not in new_manifest[key]['cached']:
                # Handle core as a special package, we already did it so we pass it to NEW data structure directly
                if key=='core':
                  print('Adding %s to %s FILES' % (item, key))
                  if item.endswith('*'):
                      wildcards.append(item)
                  if isCached(item):
                      new_manifest[key]['cached'].append(item)
                  else:
                      new_manifest[key]['files'].append(item)

                  # Check for repeated files
                  if item not in allfiles:
                      allfiles.append(item)
                  else:
                      repeated.append(item)

                else:


                    # Check if this dependency is already contained on another package, so we add it
                    # as an RDEPENDS, or if its not, it means it should be contained on the current
                    # package, so we should add it to FILES
                    for newkey in old_manifest:
                        # Debug
                        #print('Checking %s ' % item + ' in %s' % newkey)
                        if item in old_manifest[newkey]['files'] or item in old_manifest[newkey]['cached']:      
                                # Since were nesting, we need to check its not the same key
                                if(newkey!=key):
                                    if newkey not in new_manifest[key]['rdepends']:
                                       # Add it to the new manifest data struct
                                       reportRDEPS.append('Adding %s to %s RDEPENDS, because it contains %s\n' % (newkey, key, item))
                                       new_manifest[key]['rdepends'].append(newkey)
                                    break
                    else:
                      # A module shouldn't contain itself (${libdir}/python3/sqlite3 shouldnt be on sqlite3 files)
                      if os.path.basename(item) != key:
                        reportFILES.append(('Adding %s to %s FILES\n' % (item, key)))
                        # Since it wasnt found on another package, its not an RDEP, so add it to FILES for this package
                        if isCached(item):
                            new_manifest[key]['cached'].append(item)
                        else:
                            new_manifest[key]['files'].append(item)

                        if item.endswith('*'):
                            wildcards.append(item)
                        if item not in allfiles:
                            allfiles.append(item)
                        else:
                            repeated.append(item)

        print('\n')
        print('#################################')
        print('Summary for module %s' % value)
        print('FILES found for module %s:' % value)
        print(''.join(reportFILES))
        print('RDEPENDS found for module %s:' % value)
        print(''.join(reportRDEPS))
        print('#################################')

print ('The following files are repeated (contained in more than one package), please check which package should get it:')
print (repeated)
print('The following files contain wildcards, please check they are necessary')
print(wildcards)
print('The following files contain folders, please check they are necessary')
print(hasfolders)

# Sort it just so it looks nicer
for key in new_manifest:
    new_manifest[key]['files'].sort()
    new_manifest[key]['cached'].sort()
    new_manifest[key]['rdepends'].sort()

# Create the manifest from the data structure that was built
with open('python3-manifest.json.new','w') as outfile:
    json.dump(new_manifest,outfile,sort_keys=True, indent=4)
    outfile.write('\n')