poky/meta/recipes-devtools/python/python3/create_manifest3.py - stefanberger/openbmc - Gitiles

 # This script is used as a bitbake task to create a new python manifest
 # $ bitbake python -c create_manifest
 #
 # Our goal is to keep python-core as small as posible and add other python
 # packages only when the user needs them, hence why we split upstream python
 # into several packages.
 #
 # In a very simplistic way what this does is:
 # Launch python and see specifically what is required for it to run at a minimum
 #
 # Go through the python-manifest file and launch a separate task for every single
 # one of the files on each package, this task will check what was required for that
 # specific module to run, these modules will be called dependencies.
 # The output of such task will be a list of the modules or dependencies that were
 # found for that file.
 #
 # Such output will be parsed by this script, we will look for each dependency on the
 # manifest and if we find that another package already includes it, then we will add
 # that package as an RDEPENDS to the package we are currently checking; in case we dont
 # find the current dependency on any other package we will add it to the current package
 # as part of FILES.
 #
 #
 # This way we will create a new manifest from the data structure that was built during
 # this process, ont this new manifest each package will contain specifically only
 # what it needs to run.
 #
 # There are some caveats which we try to deal with, such as repeated files on different
 # packages, packages that include folders, wildcards, and special packages.
 # Its also important to note that this method only works for python files, and shared
 # libraries. Static libraries, header files and binaries need to be dealt with manually.
 #
 # This script differs from its python2 version mostly on how shared libraries are handled
 # The manifest file for python3 has an extra field which contains the cached files for
 # each package.
 # Tha method to handle cached files does not work when a module includes a folder which
 # itself contains the pycache folder, gladly this is almost never the case.
 #
 # Author: Alejandro Enedino Hernandez Samaniego "aehs29" <aehs29@gmail.com>


 import sys
 import subprocess
 import json
 import os

 # Hack to get native python search path (for folders), not fond of it but it works for now
 pivot='recipe-sysroot-native'
 for p in sys.path:
   if pivot in p:
     nativelibfolder=p[:p.find(pivot)+len(pivot)]

 # Empty dict to hold the whole manifest
 new_manifest = {}

 # Check for repeated files, folders and wildcards
 allfiles=[]
 repeated=[]
 wildcards=[]

 hasfolders=[]
 allfolders=[]

 def isFolder(value):
   if os.path.isdir(value.replace('${libdir}',nativelibfolder+'/usr/lib')) or os.path.isdir(value.replace('${libdir}',nativelibfolder+'/usr/lib64')) or os.path.isdir(value.replace('${libdir}',nativelibfolder+'/usr/lib32')):
     return True
   else:
     return False

 def isCached(item):
   if '__pycache__' in item:
     return True
   else:
     return False

 # Read existing JSON manifest
 with open('python3-manifest.json') as manifest:
   old_manifest=json.load(manifest)


 # First pass to get core-package functionality, because we base everything on the fact that core is actually working
 # Not exactly the same so it should not be a function
 print ('Getting dependencies for package: core')

 # Special call to check for core package
 output = subprocess.check_output([sys.executable, 'get_module_deps3.py', 'python-core-package']).decode('utf8')
 for item in output.split():
     # We append it so it doesnt hurt what we currently have:
     if isCached(item):
         if item not in old_manifest['core']['cached']:
             # We use the same data structure since its the one which will be used to check
             # dependencies for other packages
             old_manifest['core']['cached'].append(item)
     else:
         if item not in old_manifest['core']['files']:
             # We use the same data structure since its the one which will be used to check
             # dependencies for other packages
             old_manifest['core']['files'].append(item)

 for value in old_manifest['core']['files']:
   # Ignore folders, since we don't import those, difficult to handle multilib
   if isFolder(value):
     # Pass it directly
     if isCached(value):
         if value not in old_manifest['core']['cached']:
             old_manifest['core']['cached'].append(value)
     else:
         if value not in old_manifest['core']['files']:
             old_manifest['core']['files'].append(value)
     continue
   # Ignore binaries, since we don't import those, assume it was added correctly (manually)
   if '${bindir}' in value:
     # Pass it directly
     if value not in old_manifest['core']['files']:
       old_manifest['core']['files'].append(value)
     continue
   # Ignore empty values
   if value == '':
     continue
   if '${includedir}' in value:
     if value not in old_manifest['core']['files']:
       old_manifest['core']['files'].append(value)
     continue
   # Get module name , shouldnt be affected by libdir/bindir
   value = os.path.splitext(os.path.basename(os.path.normpath(value)))[0]

   # Launch separate task for each module for deterministic behavior
   # Each module will only import what is necessary for it to work in specific
   print ('Getting dependencies for module: %s' % value)
   output = subprocess.check_output([sys.executable, 'get_module_deps3.py', '%s' % value]).decode('utf8')
   print ('The following dependencies were found for module %s:\n' % value)
   print (output)
   for item in output.split():
     # We append it so it doesnt hurt what we currently have:
     if isCached(item):
         if item not in old_manifest['core']['cached']:
             # We use the same data structure since its the one which will be used to check
             # dependencies for other packages
             old_manifest['core']['cached'].append(item)
     else:
         if item not in old_manifest['core']['files']:
             # We use the same data structure since its the one which will be used to check
             # dependencies for other packages
             old_manifest['core']['files'].append(item)


 # We check which packages include folders
 for key in old_manifest:
     for value in old_manifest[key]['files']:
         # Ignore folders, since we don't import those, difficult to handle multilib
         if isFolder(value):
             print ('%s is a folder' % value)
             if key not in hasfolders:
                 hasfolders.append(key)
             if value not in allfolders:
                 allfolders.append(value)

 for key in old_manifest:
     # Use an empty dict as data structure to hold data for each package and fill it up
     new_manifest[key]={}
     new_manifest[key]['files']=[]

     new_manifest[key]['rdepends']=[]
     # All packages should depend on core
     if key != 'core':
          new_manifest[key]['rdepends'].append('core')
          new_manifest[key]['cached']=[]
     else:
          new_manifest[key]['cached']=old_manifest[key]['cached']
     new_manifest[key]['summary']=old_manifest[key]['summary']

     # Handle special cases, we assume that when they were manually added
     # to the manifest we knew what we were doing.
     print('\n')
     print('--------------------------')
     print ('Handling package %s' % key)
     print('--------------------------')
     special_packages=['misc', 'modules', 'dev']
     if key in special_packages or 'staticdev' in key:
         print('Passing %s package directly' % key)
         new_manifest[key]=old_manifest[key]
         continue

     for value in old_manifest[key]['files']:
         # We already handled core on the first pass
         if key == 'core':
             new_manifest[key]['files'].append(value)
             continue
         # Ignore folders, since we don't import those, difficult to handle multilib
         if isFolder(value):
             # Pass folders directly
             new_manifest[key]['files'].append(value)
         # Ignore binaries, since we don't import those
         if '${bindir}' in value:
             # Pass it directly to the new manifest data structure
             if value not in new_manifest[key]['files']:
                 new_manifest[key]['files'].append(value)
             continue
         # Ignore empty values
         if value == '':
             continue
         if '${includedir}' in value:
             if value not in new_manifest[key]['files']:
                 new_manifest[key]['files'].append(value)
             continue

         # Get module name , shouldnt be affected by libdir/bindir
         # We need to check if the imported module comes from another (e.g. sqlite3.dump)
         path,value = os.path.split(value)
         path = os.path.basename(path)
         value = os.path.splitext(os.path.basename(value))[0]

         # If this condition is met, it means we need to import it from another module
         # or its the folder itself (e.g. unittest)
         if path == key:
           if value:
             value = path + '.' + value
           else:
             value = path

         # Launch separate task for each module for deterministic behavior
         # Each module will only import what is necessary for it to work in specific
         print ('\nGetting dependencies for module: %s' % value)
         output = subprocess.check_output([sys.executable, 'get_module_deps3.py', '%s' % value]).decode('utf8')
         # We can print dependencies for debugging purposes
         print ('The following dependencies were found for module %s:\n' % value)
         print (output)
         # Output will have all dependencies

         reportFILES = []
         reportRDEPS = []

         for item in output.split():

             # Warning: This first part is ugly
             # One of the dependencies that was found, could be inside of one of the folders included by another package
             # We need to check if this happens so we can add the package containing the folder as an rdependency
             # e.g. Folder encodings contained in codecs
             # This would be solved if no packages included any folders

             # This can be done in two ways:
             # 1 - We assume that if we take out the filename from the path we would get
             #   the folder string, then we would check if folder string is in the list of folders
             #   This would not work if a package contains a folder which contains another folder
             #   e.g. path/folder1/folder2/filename  folder_string= path/folder1/folder2
             #   folder_string would not match any value contained in the list of folders
             #
             # 2 - We do it the other way around, checking if the folder is contained in the path
             #   e.g. path/folder1/folder2/filename  folder_string= path/folder1/folder2
             #   is folder_string inside path/folder1/folder2/filename?,
             #   Yes, it works, but we waste a couple of milliseconds.

             inFolders=False
             for folder in allfolders:
                 if folder in item:
                     inFolders = True # Did we find a folder?
                     folderFound = False # Second flag to break inner for
                     # Loop only through packages which contain folders
                     for keyfolder in hasfolders:
                         if (folderFound == False):
                             #print('Checking folder %s on package %s' % (item,keyfolder))
                             for file_folder in old_manifest[keyfolder]['files'] or file_folder in old_manifest[keyfolder]['cached']:
                                 if file_folder==folder:
                                     print ('%s folder found in %s' % (folder, keyfolder))
                                     folderFound = True
                                     if keyfolder not in new_manifest[key]['rdepends'] and keyfolder != key:
                                         new_manifest[key]['rdepends'].append(keyfolder)

                         else:
                             break

             # A folder was found so we're done with this item, we can go on
             if inFolders:
                 continue


             # We might already have it on the dictionary since it could depend on a (previously checked) module
             if item not in new_manifest[key]['files'] and item not in new_manifest[key]['cached']:
                 # Handle core as a special package, we already did it so we pass it to NEW data structure directly
                 if key=='core':
                   print('Adding %s to %s FILES' % (item, key))
                   if item.endswith('*'):
                       wildcards.append(item)
                   if isCached(item):
                       new_manifest[key]['cached'].append(item)
                   else:
                       new_manifest[key]['files'].append(item)

                   # Check for repeated files
                   if item not in allfiles:
                       allfiles.append(item)
                   else:
                       repeated.append(item)

                 else:


                     # Check if this dependency is already contained on another package, so we add it
                     # as an RDEPENDS, or if its not, it means it should be contained on the current
                     # package, so we should add it to FILES
                     for newkey in old_manifest:
                         # Debug
                         #print('Checking %s ' % item + ' in %s' % newkey)
                         if item in old_manifest[newkey]['files'] or item in old_manifest[newkey]['cached']:
                                 # Since were nesting, we need to check its not the same key
                                 if(newkey!=key):
                                     if newkey not in new_manifest[key]['rdepends']:
                                        # Add it to the new manifest data struct
                                        reportRDEPS.append('Adding %s to %s RDEPENDS, because it contains %s\n' % (newkey, key, item))
                                        new_manifest[key]['rdepends'].append(newkey)
                                     break
                     else:
                       # A module shouldn't contain itself (${libdir}/python3/sqlite3 shouldnt be on sqlite3 files)
                       if os.path.basename(item) != key:
                         reportFILES.append(('Adding %s to %s FILES\n' % (item, key)))
                         # Since it wasnt found on another package, its not an RDEP, so add it to FILES for this package
                         if isCached(item):
                             new_manifest[key]['cached'].append(item)
                         else:
                             new_manifest[key]['files'].append(item)

                         if item.endswith('*'):
                             wildcards.append(item)
                         if item not in allfiles:
                             allfiles.append(item)
                         else:
                             repeated.append(item)

         print('\n')
         print('#################################')
         print('Summary for module %s' % value)
         print('FILES found for module %s:' % value)
         print(''.join(reportFILES))
         print('RDEPENDS found for module %s:' % value)
         print(''.join(reportRDEPS))
         print('#################################')

 print ('The following files are repeated (contained in more than one package), please check which package should get it:')
 print (repeated)
 print('The following files contain wildcards, please check they are necessary')
 print(wildcards)
 print('The following files contain folders, please check they are necessary')
 print(hasfolders)

 # Sort it just so it looks nicer
 for key in new_manifest:
     new_manifest[key]['files'].sort()
     new_manifest[key]['cached'].sort()
     new_manifest[key]['rdepends'].sort()

 # Create the manifest from the data structure that was built
 with open('python3-manifest.json.new','w') as outfile:
     json.dump(new_manifest,outfile,sort_keys=True, indent=4)
     outfile.write('\n')
	# This script is used as a bitbake task to create a new python manifest
	# $ bitbake python -c create_manifest
	#
	# Our goal is to keep python-core as small as posible and add other python
	# packages only when the user needs them, hence why we split upstream python
	# into several packages.
	#
	# In a very simplistic way what this does is:
	# Launch python and see specifically what is required for it to run at a minimum
	#
	# Go through the python-manifest file and launch a separate task for every single
	# one of the files on each package, this task will check what was required for that
	# specific module to run, these modules will be called dependencies.
	# The output of such task will be a list of the modules or dependencies that were
	# found for that file.
	#
	# Such output will be parsed by this script, we will look for each dependency on the
	# manifest and if we find that another package already includes it, then we will add
	# that package as an RDEPENDS to the package we are currently checking; in case we dont
	# find the current dependency on any other package we will add it to the current package
	# as part of FILES.
	#
	#
	# This way we will create a new manifest from the data structure that was built during
	# this process, ont this new manifest each package will contain specifically only
	# what it needs to run.
	#
	# There are some caveats which we try to deal with, such as repeated files on different
	# packages, packages that include folders, wildcards, and special packages.
	# Its also important to note that this method only works for python files, and shared
	# libraries. Static libraries, header files and binaries need to be dealt with manually.
	#
	# This script differs from its python2 version mostly on how shared libraries are handled
	# The manifest file for python3 has an extra field which contains the cached files for
	# each package.
	# Tha method to handle cached files does not work when a module includes a folder which
	# itself contains the pycache folder, gladly this is almost never the case.
	#
	# Author: Alejandro Enedino Hernandez Samaniego "aehs29" <aehs29@gmail.com>


	import sys
	import subprocess
	import json
	import os

	# Hack to get native python search path (for folders), not fond of it but it works for now
	pivot='recipe-sysroot-native'
	for p in sys.path:
	if pivot in p:
	nativelibfolder=p[:p.find(pivot)+len(pivot)]

	# Empty dict to hold the whole manifest
	new_manifest = {}

	# Check for repeated files, folders and wildcards
	allfiles=[]
	repeated=[]
	wildcards=[]

	hasfolders=[]
	allfolders=[]

	def isFolder(value):
	if os.path.isdir(value.replace('${libdir}',nativelibfolder+'/usr/lib')) or os.path.isdir(value.replace('${libdir}',nativelibfolder+'/usr/lib64')) or os.path.isdir(value.replace('${libdir}',nativelibfolder+'/usr/lib32')):
	return True
	else:
	return False

	def isCached(item):
	if '__pycache__' in item:
	return True
	else:
	return False

	# Read existing JSON manifest
	with open('python3-manifest.json') as manifest:
	old_manifest=json.load(manifest)


	# First pass to get core-package functionality, because we base everything on the fact that core is actually working
	# Not exactly the same so it should not be a function
	print ('Getting dependencies for package: core')

	# Special call to check for core package
	output = subprocess.check_output([sys.executable, 'get_module_deps3.py', 'python-core-package']).decode('utf8')
	for item in output.split():
	# We append it so it doesnt hurt what we currently have:
	if isCached(item):
	if item not in old_manifest['core']['cached']:
	# We use the same data structure since its the one which will be used to check
	# dependencies for other packages
	old_manifest['core']['cached'].append(item)
	else:
	if item not in old_manifest['core']['files']:
	# We use the same data structure since its the one which will be used to check
	# dependencies for other packages
	old_manifest['core']['files'].append(item)

	for value in old_manifest['core']['files']:
	# Ignore folders, since we don't import those, difficult to handle multilib
	if isFolder(value):
	# Pass it directly
	if isCached(value):
	if value not in old_manifest['core']['cached']:
	old_manifest['core']['cached'].append(value)
	else:
	if value not in old_manifest['core']['files']:
	old_manifest['core']['files'].append(value)
	continue
	# Ignore binaries, since we don't import those, assume it was added correctly (manually)
	if '${bindir}' in value:
	# Pass it directly
	if value not in old_manifest['core']['files']:
	old_manifest['core']['files'].append(value)
	continue
	# Ignore empty values
	if value == '':
	continue
	if '${includedir}' in value:
	if value not in old_manifest['core']['files']:
	old_manifest['core']['files'].append(value)
	continue
	# Get module name , shouldnt be affected by libdir/bindir
	value = os.path.splitext(os.path.basename(os.path.normpath(value)))[0]

	# Launch separate task for each module for deterministic behavior
	# Each module will only import what is necessary for it to work in specific
	print ('Getting dependencies for module: %s' % value)
	output = subprocess.check_output([sys.executable, 'get_module_deps3.py', '%s' % value]).decode('utf8')
	print ('The following dependencies were found for module %s:\n' % value)
	print (output)
	for item in output.split():
	# We append it so it doesnt hurt what we currently have:
	if isCached(item):
	if item not in old_manifest['core']['cached']:
	# We use the same data structure since its the one which will be used to check
	# dependencies for other packages
	old_manifest['core']['cached'].append(item)
	else:
	if item not in old_manifest['core']['files']:
	# We use the same data structure since its the one which will be used to check
	# dependencies for other packages
	old_manifest['core']['files'].append(item)


	# We check which packages include folders
	for key in old_manifest:
	for value in old_manifest[key]['files']:
	# Ignore folders, since we don't import those, difficult to handle multilib
	if isFolder(value):
	print ('%s is a folder' % value)
	if key not in hasfolders:
	hasfolders.append(key)
	if value not in allfolders:
	allfolders.append(value)

	for key in old_manifest:
	# Use an empty dict as data structure to hold data for each package and fill it up
	new_manifest[key]={}
	new_manifest[key]['files']=[]

	new_manifest[key]['rdepends']=[]
	# All packages should depend on core
	if key != 'core':
	new_manifest[key]['rdepends'].append('core')
	new_manifest[key]['cached']=[]
	else:
	new_manifest[key]['cached']=old_manifest[key]['cached']
	new_manifest[key]['summary']=old_manifest[key]['summary']

	# Handle special cases, we assume that when they were manually added
	# to the manifest we knew what we were doing.
	print('\n')
	print('--------------------------')
	print ('Handling package %s' % key)
	print('--------------------------')
	special_packages=['misc', 'modules', 'dev']
	if key in special_packages or 'staticdev' in key:
	print('Passing %s package directly' % key)
	new_manifest[key]=old_manifest[key]
	continue

	for value in old_manifest[key]['files']:
	# We already handled core on the first pass
	if key == 'core':
	new_manifest[key]['files'].append(value)
	continue
	# Ignore folders, since we don't import those, difficult to handle multilib
	if isFolder(value):
	# Pass folders directly
	new_manifest[key]['files'].append(value)
	# Ignore binaries, since we don't import those
	if '${bindir}' in value:
	# Pass it directly to the new manifest data structure
	if value not in new_manifest[key]['files']:
	new_manifest[key]['files'].append(value)
	continue
	# Ignore empty values
	if value == '':
	continue
	if '${includedir}' in value:
	if value not in new_manifest[key]['files']:
	new_manifest[key]['files'].append(value)
	continue

	# Get module name , shouldnt be affected by libdir/bindir
	# We need to check if the imported module comes from another (e.g. sqlite3.dump)
	path,value = os.path.split(value)
	path = os.path.basename(path)
	value = os.path.splitext(os.path.basename(value))[0]

	# If this condition is met, it means we need to import it from another module
	# or its the folder itself (e.g. unittest)
	if path == key:
	if value:
	value = path + '.' + value
	else:
	value = path

	# Launch separate task for each module for deterministic behavior
	# Each module will only import what is necessary for it to work in specific
	print ('\nGetting dependencies for module: %s' % value)
	output = subprocess.check_output([sys.executable, 'get_module_deps3.py', '%s' % value]).decode('utf8')
	# We can print dependencies for debugging purposes
	print ('The following dependencies were found for module %s:\n' % value)
	print (output)
	# Output will have all dependencies

	reportFILES = []
	reportRDEPS = []

	for item in output.split():

	# Warning: This first part is ugly
	# One of the dependencies that was found, could be inside of one of the folders included by another package
	# We need to check if this happens so we can add the package containing the folder as an rdependency
	# e.g. Folder encodings contained in codecs
	# This would be solved if no packages included any folders

	# This can be done in two ways:
	# 1 - We assume that if we take out the filename from the path we would get
	# the folder string, then we would check if folder string is in the list of folders
	# This would not work if a package contains a folder which contains another folder
	# e.g. path/folder1/folder2/filename folder_string= path/folder1/folder2
	# folder_string would not match any value contained in the list of folders
	#
	# 2 - We do it the other way around, checking if the folder is contained in the path
	# e.g. path/folder1/folder2/filename folder_string= path/folder1/folder2
	# is folder_string inside path/folder1/folder2/filename?,
	# Yes, it works, but we waste a couple of milliseconds.

	inFolders=False
	for folder in allfolders:
	if folder in item:
	inFolders = True # Did we find a folder?
	folderFound = False # Second flag to break inner for
	# Loop only through packages which contain folders
	for keyfolder in hasfolders:
	if (folderFound == False):
	#print('Checking folder %s on package %s' % (item,keyfolder))
	for file_folder in old_manifest[keyfolder]['files'] or file_folder in old_manifest[keyfolder]['cached']:
	if file_folder==folder:
	print ('%s folder found in %s' % (folder, keyfolder))
	folderFound = True
	if keyfolder not in new_manifest[key]['rdepends'] and keyfolder != key:
	new_manifest[key]['rdepends'].append(keyfolder)

	else:
	break

	# A folder was found so we're done with this item, we can go on
	if inFolders:
	continue


	# We might already have it on the dictionary since it could depend on a (previously checked) module
	if item not in new_manifest[key]['files'] and item not in new_manifest[key]['cached']:
	# Handle core as a special package, we already did it so we pass it to NEW data structure directly
	if key=='core':
	print('Adding %s to %s FILES' % (item, key))
	if item.endswith('*'):
	wildcards.append(item)
	if isCached(item):
	new_manifest[key]['cached'].append(item)
	else:
	new_manifest[key]['files'].append(item)

	# Check for repeated files
	if item not in allfiles:
	allfiles.append(item)
	else:
	repeated.append(item)

	else:


	# Check if this dependency is already contained on another package, so we add it
	# as an RDEPENDS, or if its not, it means it should be contained on the current
	# package, so we should add it to FILES
	for newkey in old_manifest:
	# Debug
	#print('Checking %s ' % item + ' in %s' % newkey)
	if item in old_manifest[newkey]['files'] or item in old_manifest[newkey]['cached']:
	# Since were nesting, we need to check its not the same key
	if(newkey!=key):
	if newkey not in new_manifest[key]['rdepends']:
	# Add it to the new manifest data struct
	reportRDEPS.append('Adding %s to %s RDEPENDS, because it contains %s\n' % (newkey, key, item))
	new_manifest[key]['rdepends'].append(newkey)
	break
	else:
	# A module shouldn't contain itself (${libdir}/python3/sqlite3 shouldnt be on sqlite3 files)
	if os.path.basename(item) != key:
	reportFILES.append(('Adding %s to %s FILES\n' % (item, key)))
	# Since it wasnt found on another package, its not an RDEP, so add it to FILES for this package
	if isCached(item):
	new_manifest[key]['cached'].append(item)
	else:
	new_manifest[key]['files'].append(item)

	if item.endswith('*'):
	wildcards.append(item)
	if item not in allfiles:
	allfiles.append(item)
	else:
	repeated.append(item)

	print('\n')
	print('#################################')
	print('Summary for module %s' % value)
	print('FILES found for module %s:' % value)
	print(''.join(reportFILES))
	print('RDEPENDS found for module %s:' % value)
	print(''.join(reportRDEPS))
	print('#################################')

	print ('The following files are repeated (contained in more than one package), please check which package should get it:')
	print (repeated)
	print('The following files contain wildcards, please check they are necessary')
	print(wildcards)
	print('The following files contain folders, please check they are necessary')
	print(hasfolders)

	# Sort it just so it looks nicer
	for key in new_manifest:
	new_manifest[key]['files'].sort()
	new_manifest[key]['cached'].sort()
	new_manifest[key]['rdepends'].sort()

	# Create the manifest from the data structure that was built
	with open('python3-manifest.json.new','w') as outfile:
	json.dump(new_manifest,outfile,sort_keys=True, indent=4)
	outfile.write('\n')