Brad Bishop | 316dfdd | 2018-06-25 12:45:53 -0400 | [diff] [blame^] | 1 | # This script is used as a bitbake task to create a new python manifest |
| 2 | # $ bitbake python -c create_manifest |
| 3 | # |
| 4 | # Our goal is to keep python-core as small as posible and add other python |
| 5 | # packages only when the user needs them, hence why we split upstream python |
| 6 | # into several packages. |
| 7 | # |
| 8 | # In a very simplistic way what this does is: |
| 9 | # Launch python and see specifically what is required for it to run at a minimum |
| 10 | # |
| 11 | # Go through the python-manifest file and launch a separate task for every single |
| 12 | # one of the files on each package, this task will check what was required for that |
| 13 | # specific module to run, these modules will be called dependencies. |
| 14 | # The output of such task will be a list of the modules or dependencies that were |
| 15 | # found for that file. |
| 16 | # |
| 17 | # Such output will be parsed by this script, we will look for each dependency on the |
| 18 | # manifest and if we find that another package already includes it, then we will add |
| 19 | # that package as an RDEPENDS to the package we are currently checking; in case we dont |
| 20 | # find the current dependency on any other package we will add it to the current package |
| 21 | # as part of FILES. |
| 22 | # |
| 23 | # |
| 24 | # This way we will create a new manifest from the data structure that was built during |
| 25 | # this process, ont this new manifest each package will contain specifically only |
| 26 | # what it needs to run. |
| 27 | # |
| 28 | # There are some caveats which we try to deal with, such as repeated files on different |
| 29 | # packages, packages that include folders, wildcards, and special packages. |
| 30 | # Its also important to note that this method only works for python files, and shared |
| 31 | # libraries. Static libraries, header files and binaries need to be dealt with manually. |
| 32 | # |
| 33 | # Author: Alejandro Enedino Hernandez Samaniego "aehs29" <aehs29@gmail.com> |
| 34 | |
| 35 | |
| 36 | import sys |
| 37 | import subprocess |
| 38 | import json |
| 39 | import os |
| 40 | |
| 41 | # Hack to get native python search path (for folders), not fond of it but it works for now |
| 42 | pivot='recipe-sysroot-native' |
| 43 | for p in sys.path: |
| 44 | if pivot in p: |
| 45 | nativelibfolder=p[:p.find(pivot)+len(pivot)] |
| 46 | |
| 47 | # Empty dict to hold the whole manifest |
| 48 | new_manifest = {} |
| 49 | |
| 50 | # Check for repeated files, folders and wildcards |
| 51 | allfiles=[] |
| 52 | repeated=[] |
| 53 | wildcards=[] |
| 54 | |
| 55 | hasfolders=[] |
| 56 | allfolders=[] |
| 57 | |
| 58 | def isFolder(value): |
| 59 | if os.path.isdir(value.replace('${libdir}',nativelibfolder+'/usr/lib')) or os.path.isdir(value.replace('${libdir}',nativelibfolder+'/usr/lib64')) or os.path.isdir(value.replace('${libdir}',nativelibfolder+'/usr/lib32')): |
| 60 | return True |
| 61 | else: |
| 62 | return False |
| 63 | |
| 64 | # Read existing JSON manifest |
| 65 | with open('python2-manifest.json') as manifest: |
| 66 | old_manifest=json.load(manifest) |
| 67 | |
| 68 | |
| 69 | # First pass to get core-package functionality, because we base everything on the fact that core is actually working |
| 70 | # Not exactly the same so it should not be a function |
| 71 | print ("Getting dependencies for core package:") |
| 72 | |
| 73 | # Special call to check for core package |
| 74 | output = subprocess.check_output([sys.executable, 'get_module_deps2.py', 'python-core-package']) |
| 75 | for item in output.split(): |
| 76 | # We append it so it doesnt hurt what we currently have: |
| 77 | if item not in old_manifest['core']['files']: |
| 78 | # We use the same data structure since its the one which will be used to check |
| 79 | # dependencies for other packages |
| 80 | old_manifest['core']['files'].append(item) |
| 81 | |
| 82 | for value in old_manifest['core']['files']: |
| 83 | # Ignore folders, since we don't import those, difficult to handle multilib |
| 84 | if isFolder(value): |
| 85 | # Pass it directly |
| 86 | if value not in old_manifest['core']['files']: |
| 87 | old_manifest['core']['files'].append(value) |
| 88 | # Ignore binaries, since we don't import those, assume it was added correctly (manually) |
| 89 | if '${bindir}' in value: |
| 90 | # Pass it directly |
| 91 | if value not in old_manifest['core']['files']: |
| 92 | old_manifest['core']['files'].append(value) |
| 93 | continue |
| 94 | # Ignore empty values |
| 95 | if value == '': |
| 96 | continue |
| 97 | if '${includedir}' in value: |
| 98 | if value not in old_manifest['core']['files']: |
| 99 | old_manifest['core']['files'].append(value) |
| 100 | continue |
| 101 | # Get module name , shouldnt be affected by libdir/bindir |
| 102 | value = os.path.splitext(os.path.basename(os.path.normpath(value)))[0] |
| 103 | |
| 104 | |
| 105 | # Launch separate task for each module for deterministic behavior |
| 106 | # Each module will only import what is necessary for it to work in specific |
| 107 | print ('Getting dependencies for module: %s' % value) |
| 108 | output = subprocess.check_output([sys.executable, 'get_module_deps2.py', '%s' % value]) |
| 109 | for item in output.split(): |
| 110 | # We append it so it doesnt hurt what we currently have: |
| 111 | if item not in old_manifest['core']['files']: |
| 112 | old_manifest['core']['files'].append(item) |
| 113 | |
| 114 | # We check which packages include folders |
| 115 | for key in old_manifest: |
| 116 | for value in old_manifest[key]['files']: |
| 117 | # Ignore folders, since we don't import those, difficult to handle multilib |
| 118 | if isFolder(value): |
| 119 | print ('%s is a folder' % value) |
| 120 | if key not in hasfolders: |
| 121 | hasfolders.append(key) |
| 122 | if value not in allfolders: |
| 123 | allfolders.append(value) |
| 124 | |
| 125 | for key in old_manifest: |
| 126 | # Use an empty dict as data structure to hold data for each package and fill it up |
| 127 | new_manifest[key]={} |
| 128 | new_manifest[key]['files']=[] |
| 129 | new_manifest[key]['rdepends']=[] |
| 130 | # All packages should depend on core |
| 131 | if key != 'core': |
| 132 | new_manifest[key]['rdepends'].append('core') |
| 133 | new_manifest[key]['summary']=old_manifest[key]['summary'] |
| 134 | |
| 135 | # Handle special cases, we assume that when they were manually added |
| 136 | # to the manifest we knew what we were doing. |
| 137 | print ('Handling package %s' % key) |
| 138 | special_packages=['misc', 'modules', 'dev'] |
| 139 | if key in special_packages or 'staticdev' in key: |
| 140 | print('Passing %s package directly' % key) |
| 141 | new_manifest[key]=old_manifest[key] |
| 142 | continue |
| 143 | |
| 144 | for value in old_manifest[key]['files']: |
| 145 | # We already handled core on the first pass |
| 146 | if key == 'core': |
| 147 | new_manifest[key]['files'].append(value) |
| 148 | continue |
| 149 | # Ignore folders, since we don't import those, difficult to handle multilib |
| 150 | if isFolder(value): |
| 151 | # Pass folders directly |
| 152 | new_manifest[key]['files'].append(value) |
| 153 | # Ignore binaries, since we don't import those |
| 154 | if '${bindir}' in value: |
| 155 | # Pass it directly to the new manifest data structure |
| 156 | if value not in new_manifest[key]['files']: |
| 157 | new_manifest[key]['files'].append(value) |
| 158 | continue |
| 159 | # Ignore empty values |
| 160 | if value == '': |
| 161 | continue |
| 162 | if '${includedir}' in value: |
| 163 | if value not in new_manifest[key]['files']: |
| 164 | new_manifest[key]['files'].append(value) |
| 165 | continue |
| 166 | # Get module name , shouldnt be affected by libdir/bindir |
| 167 | value = os.path.splitext(os.path.basename(os.path.normpath(value)))[0] |
| 168 | |
| 169 | # Launch separate task for each module for deterministic behavior |
| 170 | # Each module will only import what is necessary for it to work in specific |
| 171 | print ('Getting dependencies for module: %s' % value) |
| 172 | output = subprocess.check_output([sys.executable, 'get_module_deps2.py', '%s' % value]) |
| 173 | |
| 174 | # We can print dependencies for debugging purposes |
| 175 | #print (output) |
| 176 | # Output will have all dependencies |
| 177 | for item in output.split(): |
| 178 | |
| 179 | # Warning: This first part is ugly |
| 180 | # One of the dependencies that was found, could be inside of one of the folders included by another package |
| 181 | # We need to check if this happens so we can add the package containing the folder as an RDEPENDS |
| 182 | # e.g. Folder encodings contained in codecs |
| 183 | # This would be solved if no packages included any folders |
| 184 | |
| 185 | # This can be done in two ways: |
| 186 | # 1 - We assume that if we take out the filename from the path we would get |
| 187 | # the folder string, then we would check if folder string is in the list of folders |
| 188 | # This would not work if a package contains a folder which contains another folder |
| 189 | # e.g. path/folder1/folder2/filename folder_string= path/folder1/folder2 |
| 190 | # folder_string would not match any value contained in the list of folders |
| 191 | # |
| 192 | # 2 - We do it the other way around, checking if the folder is contained in the path |
| 193 | # e.g. path/folder1/folder2/filename folder_string= path/folder1/folder2 |
| 194 | # is folder_string inside path/folder1/folder2/filename?, |
| 195 | # Yes, it works, but we waste a couple of milliseconds. |
| 196 | |
| 197 | inFolders=False |
| 198 | for folder in allfolders: |
| 199 | if folder in item: |
| 200 | inFolders = True # Did we find a folder? |
| 201 | folderFound = False # Second flag to break inner for |
| 202 | # Loop only through packages which contain folders |
| 203 | for keyfolder in hasfolders: |
| 204 | if (folderFound == False): |
| 205 | #print("Checking folder %s on package %s" % (item,keyfolder)) |
| 206 | for file_folder in old_manifest[keyfolder]['files']: |
| 207 | if file_folder==folder: |
| 208 | print ('%s found in %s' % (folder, keyfolder)) |
| 209 | folderFound = True |
| 210 | if keyfolder not in new_manifest[key]['rdepends'] and keyfolder != key: |
| 211 | new_manifest[key]['rdepends'].append(keyfolder) |
| 212 | else: |
| 213 | break |
| 214 | |
| 215 | # A folder was found so we're done with this item, we can go on |
| 216 | if inFolders: |
| 217 | continue |
| 218 | |
| 219 | # We might already have it on the dictionary since it could depend on a (previously checked) module |
| 220 | if item not in new_manifest[key]['files']: |
| 221 | # Handle core as a special package, we already did it so we pass it to NEW data structure directly |
| 222 | if key=='core': |
| 223 | print('Adding %s to %s FILES' % (item, key)) |
| 224 | if item.endswith('*'): |
| 225 | wildcards.append(item) |
| 226 | new_manifest[key]['files'].append(item) |
| 227 | |
| 228 | # Check for repeated files |
| 229 | if item not in allfiles: |
| 230 | allfiles.append(item) |
| 231 | else: |
| 232 | repeated.append(item) |
| 233 | |
| 234 | else: |
| 235 | |
| 236 | # Check if this dependency is already contained on another package, so we add it |
| 237 | # as an RDEPENDS, or if its not, it means it should be contained on the current |
| 238 | # package, so we should add it to FILES |
| 239 | for newkey in old_manifest: |
| 240 | # Debug |
| 241 | #print("Checking %s " % item + " in %s" % newkey) |
| 242 | if item in old_manifest[newkey]['files']: |
| 243 | # Since were nesting, we need to check its not the same key |
| 244 | if(newkey!=key): |
| 245 | if newkey not in new_manifest[key]['rdepends']: |
| 246 | # Add it to the new manifest data struct |
| 247 | # Debug |
| 248 | print('Adding %s to %s RDEPENDS, because it contains %s' % (newkey, key, item)) |
| 249 | new_manifest[key]['rdepends'].append(newkey) |
| 250 | break |
| 251 | else: |
| 252 | # Debug |
| 253 | print('Adding %s to %s FILES' % (item, key)) |
| 254 | # Since it wasnt found on another package, its not an RDEP, so add it to FILES for this package |
| 255 | new_manifest[key]['files'].append(item) |
| 256 | if item.endswith('*'): |
| 257 | wildcards.append(item) |
| 258 | if item not in allfiles: |
| 259 | allfiles.append(item) |
| 260 | else: |
| 261 | repeated.append(item) |
| 262 | |
| 263 | print ('The following files are repeated (contained in more than one package), please check which package should get it:') |
| 264 | print (repeated) |
| 265 | print('The following files contain wildcards, please check they are necessary') |
| 266 | print(wildcards) |
| 267 | print('The following files contain folders, please check they are necessary') |
| 268 | print(hasfolders) |
| 269 | |
| 270 | # Sort it just so it looks nice |
| 271 | for key in new_manifest: |
| 272 | new_manifest[key]['files'].sort() |
| 273 | new_manifest[key]['rdepends'].sort() |
| 274 | |
| 275 | # Create the manifest from the data structure that was built |
| 276 | with open('python2-manifest.json.new','w') as outfile: |
| 277 | json.dump(new_manifest,outfile,sort_keys=True, indent=4, separators=(',', ': ')) |