blob: a04e44d35a3afb806aeb699552c510fb3a54fc28 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001#!/usr/bin/env python
2
3# OpenEmbedded pkgdata utility
4#
5# Written by: Paul Eggleton <paul.eggleton@linux.intel.com>
6#
7# Copyright 2012-2015 Intel Corporation
8#
9# This program is free software; you can redistribute it and/or modify
10# it under the terms of the GNU General Public License version 2 as
11# published by the Free Software Foundation.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the GNU General Public License along
19# with this program; if not, write to the Free Software Foundation, Inc.,
20# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
21#
22
23import sys
24import os
25import os.path
26import fnmatch
27import re
28import argparse
29import logging
30from collections import defaultdict, OrderedDict
31
32scripts_path = os.path.dirname(os.path.realpath(__file__))
33lib_path = scripts_path + '/lib'
34sys.path = sys.path + [lib_path]
35import scriptutils
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050036import argparse_oe
Patrick Williamsc124f4f2015-09-15 14:41:29 -050037logger = scriptutils.logger_create('pkgdatautil')
38
39def tinfoil_init():
40 import bb.tinfoil
41 import logging
42 tinfoil = bb.tinfoil.Tinfoil()
43 tinfoil.prepare(True)
44
45 tinfoil.logger.setLevel(logging.WARNING)
46 return tinfoil
47
48
49def glob(args):
50 # Handle both multiple arguments and multiple values within an arg (old syntax)
51 globs = []
52 for globitem in args.glob:
53 globs.extend(globitem.split())
54
55 if not os.path.exists(args.pkglistfile):
56 logger.error('Unable to find package list file %s' % args.pkglistfile)
57 sys.exit(1)
58
59 skipval = "-locale-|^locale-base-|-dev$|-doc$|-dbg$|-staticdev$|^kernel-module-"
60 if args.exclude:
61 skipval += "|" + args.exclude
62 skipregex = re.compile(skipval)
63
Patrick Williamsf1e5d692016-03-30 15:21:19 -050064 skippedpkgs = set()
Patrick Williamsc124f4f2015-09-15 14:41:29 -050065 mappedpkgs = set()
66 with open(args.pkglistfile, 'r') as f:
67 for line in f:
68 fields = line.rstrip().split()
69 if not fields:
70 continue
71 pkg = fields[0]
72 # We don't care about other args (used to need the package architecture but the
73 # new pkgdata structure avoids the need for that)
74
75 # Skip packages for which there is no point applying globs
76 if skipregex.search(pkg):
77 logger.debug("%s -> !!" % pkg)
Patrick Williamsf1e5d692016-03-30 15:21:19 -050078 skippedpkgs.add(pkg)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050079 continue
80
81 # Skip packages that already match the globs, so if e.g. a dev package
82 # is already installed and thus in the list, we don't process it any further
83 # Most of these will be caught by skipregex already, but just in case...
84 already = False
85 for g in globs:
86 if fnmatch.fnmatchcase(pkg, g):
87 already = True
88 break
89 if already:
Patrick Williamsf1e5d692016-03-30 15:21:19 -050090 skippedpkgs.add(pkg)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050091 logger.debug("%s -> !" % pkg)
92 continue
93
94 # Define some functions
95 def revpkgdata(pkgn):
96 return os.path.join(args.pkgdata_dir, "runtime-reverse", pkgn)
97 def fwdpkgdata(pkgn):
98 return os.path.join(args.pkgdata_dir, "runtime", pkgn)
99 def readpn(pkgdata_file):
100 pn = ""
101 with open(pkgdata_file, 'r') as f:
102 for line in f:
103 if line.startswith("PN:"):
104 pn = line.split(': ')[1].rstrip()
105 return pn
106 def readrenamed(pkgdata_file):
107 renamed = ""
108 pn = os.path.basename(pkgdata_file)
109 with open(pkgdata_file, 'r') as f:
110 for line in f:
111 if line.startswith("PKG_%s:" % pn):
112 renamed = line.split(': ')[1].rstrip()
113 return renamed
114
115 # Main processing loop
116 for g in globs:
117 mappedpkg = ""
118 # First just try substitution (i.e. packagename -> packagename-dev)
119 newpkg = g.replace("*", pkg)
120 revlink = revpkgdata(newpkg)
121 if os.path.exists(revlink):
122 mappedpkg = os.path.basename(os.readlink(revlink))
123 fwdfile = fwdpkgdata(mappedpkg)
124 if os.path.exists(fwdfile):
125 mappedpkg = readrenamed(fwdfile)
126 if not os.path.exists(fwdfile + ".packaged"):
127 mappedpkg = ""
128 else:
129 revlink = revpkgdata(pkg)
130 if os.path.exists(revlink):
131 # Check if we can map after undoing the package renaming (by resolving the symlink)
132 origpkg = os.path.basename(os.readlink(revlink))
133 newpkg = g.replace("*", origpkg)
134 fwdfile = fwdpkgdata(newpkg)
135 if os.path.exists(fwdfile):
136 mappedpkg = readrenamed(fwdfile)
137 else:
138 # That didn't work, so now get the PN, substitute that, then map in the other direction
139 pn = readpn(revlink)
140 newpkg = g.replace("*", pn)
141 fwdfile = fwdpkgdata(newpkg)
142 if os.path.exists(fwdfile):
143 mappedpkg = readrenamed(fwdfile)
144 if not os.path.exists(fwdfile + ".packaged"):
145 mappedpkg = ""
146 else:
147 # Package doesn't even exist...
148 logger.debug("%s is not a valid package!" % (pkg))
149 break
150
151 if mappedpkg:
152 logger.debug("%s (%s) -> %s" % (pkg, g, mappedpkg))
153 mappedpkgs.add(mappedpkg)
154 else:
155 logger.debug("%s (%s) -> ?" % (pkg, g))
156
157 logger.debug("------")
158
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500159 print("\n".join(mappedpkgs - skippedpkgs))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500160
161def read_value(args):
162 # Handle both multiple arguments and multiple values within an arg (old syntax)
163 packages = []
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500164 if args.file:
165 with open(args.file, 'r') as f:
166 for line in f:
167 splitline = line.split()
168 if splitline:
169 packages.append(splitline[0])
170 else:
171 for pkgitem in args.pkg:
172 packages.extend(pkgitem.split())
173 if not packages:
174 logger.error("No packages specified")
175 sys.exit(1)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500176
177 def readvar(pkgdata_file, valuename):
178 val = ""
179 with open(pkgdata_file, 'r') as f:
180 for line in f:
181 if line.startswith(valuename + ":"):
182 val = line.split(': ', 1)[1].rstrip()
183 return val
184
185 logger.debug("read-value('%s', '%s' '%s'" % (args.pkgdata_dir, args.valuename, packages))
186 for package in packages:
187 pkg_split = package.split('_')
188 pkg_name = pkg_split[0]
189 logger.debug("package: '%s'" % pkg_name)
190 revlink = os.path.join(args.pkgdata_dir, "runtime-reverse", pkg_name)
191 logger.debug(revlink)
192 if os.path.exists(revlink):
193 mappedpkg = os.path.basename(os.readlink(revlink))
194 qvar = args.valuename
195 if qvar == "PKGSIZE":
196 # append packagename
197 qvar = "%s_%s" % (args.valuename, mappedpkg)
198 # PKGSIZE is now in bytes, but we we want it in KB
199 pkgsize = (int(readvar(revlink, qvar)) + 1024 // 2) // 1024
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500200 value = "%d" % pkgsize
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500201 else:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500202 value = readvar(revlink, qvar)
203 if args.prefix_name:
204 print('%s %s' % (pkg_name, value))
205 else:
206 print(value)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500207
208def lookup_pkglist(pkgs, pkgdata_dir, reverse):
209 if reverse:
210 mappings = OrderedDict()
211 for pkg in pkgs:
212 revlink = os.path.join(pkgdata_dir, "runtime-reverse", pkg)
213 logger.debug(revlink)
214 if os.path.exists(revlink):
215 mappings[pkg] = os.path.basename(os.readlink(revlink))
216 else:
217 mappings = defaultdict(list)
218 for pkg in pkgs:
219 pkgfile = os.path.join(pkgdata_dir, 'runtime', pkg)
220 if os.path.exists(pkgfile):
221 with open(pkgfile, 'r') as f:
222 for line in f:
223 fields = line.rstrip().split(': ')
224 if fields[0] == 'PKG_%s' % pkg:
225 mappings[pkg].append(fields[1])
226 break
227 return mappings
228
229def lookup_pkg(args):
230 # Handle both multiple arguments and multiple values within an arg (old syntax)
231 pkgs = []
232 for pkgitem in args.pkg:
233 pkgs.extend(pkgitem.split())
234
235 mappings = lookup_pkglist(pkgs, args.pkgdata_dir, args.reverse)
236
237 if len(mappings) < len(pkgs):
238 missing = list(set(pkgs) - set(mappings.keys()))
239 logger.error("The following packages could not be found: %s" % ', '.join(missing))
240 sys.exit(1)
241
242 if args.reverse:
243 items = mappings.values()
244 else:
245 items = []
246 for pkg in pkgs:
247 items.extend(mappings.get(pkg, []))
248
249 print('\n'.join(items))
250
251def lookup_recipe(args):
252 # Handle both multiple arguments and multiple values within an arg (old syntax)
253 pkgs = []
254 for pkgitem in args.pkg:
255 pkgs.extend(pkgitem.split())
256
257 mappings = defaultdict(list)
258 for pkg in pkgs:
259 pkgfile = os.path.join(args.pkgdata_dir, 'runtime-reverse', pkg)
260 if os.path.exists(pkgfile):
261 with open(pkgfile, 'r') as f:
262 for line in f:
263 fields = line.rstrip().split(': ')
264 if fields[0] == 'PN':
265 mappings[pkg].append(fields[1])
266 break
267 if len(mappings) < len(pkgs):
268 missing = list(set(pkgs) - set(mappings.keys()))
269 logger.error("The following packages could not be found: %s" % ', '.join(missing))
270 sys.exit(1)
271
272 items = []
273 for pkg in pkgs:
274 items.extend(mappings.get(pkg, []))
275 print('\n'.join(items))
276
277def get_recipe_pkgs(pkgdata_dir, recipe, unpackaged):
278 recipedatafile = os.path.join(pkgdata_dir, recipe)
279 if not os.path.exists(recipedatafile):
280 logger.error("Unable to find packaged recipe with name %s" % recipe)
281 sys.exit(1)
282 packages = []
283 with open(recipedatafile, 'r') as f:
284 for line in f:
285 fields = line.rstrip().split(': ')
286 if fields[0] == 'PACKAGES':
287 packages = fields[1].split()
288 break
289
290 if not unpackaged:
291 pkglist = []
292 for pkg in packages:
293 if os.path.exists(os.path.join(pkgdata_dir, 'runtime', '%s.packaged' % pkg)):
294 pkglist.append(pkg)
295 return pkglist
296 else:
297 return packages
298
299def list_pkgs(args):
300 found = False
301
302 def matchpkg(pkg):
303 if args.pkgspec:
304 matched = False
305 for pkgspec in args.pkgspec:
306 if fnmatch.fnmatchcase(pkg, pkgspec):
307 matched = True
308 break
309 if not matched:
310 return False
311 if not args.unpackaged:
312 if args.runtime:
313 revlink = os.path.join(args.pkgdata_dir, "runtime-reverse", pkg)
314 if os.path.exists(revlink):
315 # We're unlikely to get here if the package was not packaged, but just in case
316 # we add the symlinks for unpackaged files in the future
317 mappedpkg = os.path.basename(os.readlink(revlink))
318 if not os.path.exists(os.path.join(args.pkgdata_dir, 'runtime', '%s.packaged' % mappedpkg)):
319 return False
320 else:
321 return False
322 else:
323 if not os.path.exists(os.path.join(args.pkgdata_dir, 'runtime', '%s.packaged' % pkg)):
324 return False
325 return True
326
327 if args.recipe:
328 packages = get_recipe_pkgs(args.pkgdata_dir, args.recipe, args.unpackaged)
329
330 if args.runtime:
331 pkglist = []
332 runtime_pkgs = lookup_pkglist(packages, args.pkgdata_dir, False)
333 for rtpkgs in runtime_pkgs.values():
334 pkglist.extend(rtpkgs)
335 else:
336 pkglist = packages
337
338 for pkg in pkglist:
339 if matchpkg(pkg):
340 found = True
341 print("%s" % pkg)
342 else:
343 if args.runtime:
344 searchdir = 'runtime-reverse'
345 else:
346 searchdir = 'runtime'
347
348 for root, dirs, files in os.walk(os.path.join(args.pkgdata_dir, searchdir)):
349 for fn in files:
350 if fn.endswith('.packaged'):
351 continue
352 if matchpkg(fn):
353 found = True
354 print("%s" % fn)
355 if not found:
356 if args.pkgspec:
357 logger.error("Unable to find any package matching %s" % args.pkgspec)
358 else:
359 logger.error("No packages found")
360 sys.exit(1)
361
362def list_pkg_files(args):
363 import json
364
365 if args.recipe:
366 if args.pkg:
367 logger.error("list-pkg-files: If -p/--recipe is specified then a package name cannot be specified")
368 sys.exit(1)
369 recipepkglist = get_recipe_pkgs(args.pkgdata_dir, args.recipe, args.unpackaged)
370 if args.runtime:
371 pkglist = []
372 runtime_pkgs = lookup_pkglist(recipepkglist, args.pkgdata_dir, False)
373 for rtpkgs in runtime_pkgs.values():
374 pkglist.extend(rtpkgs)
375 else:
376 pkglist = recipepkglist
377 else:
378 if not args.pkg:
379 logger.error("list-pkg-files: If -p/--recipe is not specified then at least one package name must be specified")
380 sys.exit(1)
381 pkglist = args.pkg
382
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500383 for pkg in sorted(pkglist):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500384 print("%s:" % pkg)
385 if args.runtime:
386 pkgdatafile = os.path.join(args.pkgdata_dir, "runtime-reverse", pkg)
387 if not os.path.exists(pkgdatafile):
388 if args.recipe:
389 # This package was empty and thus never packaged, ignore
390 continue
391 logger.error("Unable to find any built runtime package named %s" % pkg)
392 sys.exit(1)
393 else:
394 pkgdatafile = os.path.join(args.pkgdata_dir, "runtime", pkg)
395 if not os.path.exists(pkgdatafile):
396 logger.error("Unable to find any built recipe-space package named %s" % pkg)
397 sys.exit(1)
398
399 with open(pkgdatafile, 'r') as f:
400 found = False
401 for line in f:
402 if line.startswith('FILES_INFO:'):
403 found = True
404 val = line.split(':', 1)[1].strip()
405 dictval = json.loads(val)
406 for fullpth in sorted(dictval):
407 print("\t%s" % fullpth)
408 break
409 if not found:
410 logger.error("Unable to find FILES_INFO entry in %s" % pkgdatafile)
411 sys.exit(1)
412
413def find_path(args):
414 import json
415
416 found = False
417 for root, dirs, files in os.walk(os.path.join(args.pkgdata_dir, 'runtime')):
418 for fn in files:
419 with open(os.path.join(root,fn)) as f:
420 for line in f:
421 if line.startswith('FILES_INFO:'):
422 val = line.split(':', 1)[1].strip()
423 dictval = json.loads(val)
424 for fullpth in dictval.keys():
425 if fnmatch.fnmatchcase(fullpth, args.targetpath):
426 found = True
427 print("%s: %s" % (fn, fullpth))
428 break
429 if not found:
430 logger.error("Unable to find any package producing path %s" % args.targetpath)
431 sys.exit(1)
432
433
434def main():
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500435 parser = argparse_oe.ArgumentParser(description="OpenEmbedded pkgdata tool - queries the pkgdata files written out during do_package",
436 epilog="Use %(prog)s <subcommand> --help to get help on a specific command")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500437 parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true')
438 parser.add_argument('-p', '--pkgdata-dir', help='Path to pkgdata directory (determined automatically if not specified)')
439 subparsers = parser.add_subparsers(title='subcommands', metavar='<subcommand>')
440
441 parser_lookup_pkg = subparsers.add_parser('lookup-pkg',
442 help='Translate between recipe-space package names and runtime package names',
443 description='Looks up the specified recipe-space package name(s) to see what the final runtime package name is (e.g. glibc becomes libc6), or with -r/--reverse looks up the other way.')
444 parser_lookup_pkg.add_argument('pkg', nargs='+', help='Package name to look up')
445 parser_lookup_pkg.add_argument('-r', '--reverse', help='Switch to looking up recipe-space package names from runtime package names', action='store_true')
446 parser_lookup_pkg.set_defaults(func=lookup_pkg)
447
448 parser_list_pkgs = subparsers.add_parser('list-pkgs',
449 help='List packages',
450 description='Lists packages that have been built')
451 parser_list_pkgs.add_argument('pkgspec', nargs='*', help='Package name to search for (wildcards * ? allowed, use quotes to avoid shell expansion)')
452 parser_list_pkgs.add_argument('-r', '--runtime', help='Show runtime package names instead of recipe-space package names', action='store_true')
453 parser_list_pkgs.add_argument('-p', '--recipe', help='Limit to packages produced by the specified recipe')
454 parser_list_pkgs.add_argument('-u', '--unpackaged', help='Include unpackaged (i.e. empty) packages', action='store_true')
455 parser_list_pkgs.set_defaults(func=list_pkgs)
456
457 parser_list_pkg_files = subparsers.add_parser('list-pkg-files',
458 help='List files within a package',
459 description='Lists files included in one or more packages')
460 parser_list_pkg_files.add_argument('pkg', nargs='*', help='Package name to report on (if -p/--recipe is not specified)')
461 parser_list_pkg_files.add_argument('-r', '--runtime', help='Specified package(s) are runtime package names instead of recipe-space package names', action='store_true')
462 parser_list_pkg_files.add_argument('-p', '--recipe', help='Report on all packages produced by the specified recipe')
463 parser_list_pkg_files.add_argument('-u', '--unpackaged', help='Include unpackaged (i.e. empty) packages (only useful with -p/--recipe)', action='store_true')
464 parser_list_pkg_files.set_defaults(func=list_pkg_files)
465
466 parser_lookup_recipe = subparsers.add_parser('lookup-recipe',
467 help='Find recipe producing one or more packages',
468 description='Looks up the specified runtime package(s) to see which recipe they were produced by')
469 parser_lookup_recipe.add_argument('pkg', nargs='+', help='Runtime package name to look up')
470 parser_lookup_recipe.set_defaults(func=lookup_recipe)
471
472 parser_find_path = subparsers.add_parser('find-path',
473 help='Find package providing a target path',
474 description='Finds the recipe-space package providing the specified target path')
475 parser_find_path.add_argument('targetpath', help='Path to find (wildcards * ? allowed, use quotes to avoid shell expansion)')
476 parser_find_path.set_defaults(func=find_path)
477
478 parser_read_value = subparsers.add_parser('read-value',
479 help='Read any pkgdata value for one or more packages',
480 description='Reads the named value from the pkgdata files for the specified packages')
481 parser_read_value.add_argument('valuename', help='Name of the value to look up')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500482 parser_read_value.add_argument('pkg', nargs='*', help='Runtime package name to look up')
483 parser_read_value.add_argument('-f', '--file', help='Read package names from the specified file (one per line, first field only)')
484 parser_read_value.add_argument('-n', '--prefix-name', help='Prefix output with package name', action='store_true')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500485 parser_read_value.set_defaults(func=read_value)
486
487 parser_glob = subparsers.add_parser('glob',
488 help='Expand package name glob expression',
489 description='Expands one or more glob expressions over the packages listed in pkglistfile')
490 parser_glob.add_argument('pkglistfile', help='File listing packages (one package name per line)')
491 parser_glob.add_argument('glob', nargs="+", help='Glob expression for package names, e.g. *-dev')
492 parser_glob.add_argument('-x', '--exclude', help='Exclude packages matching specified regex from the glob operation')
493 parser_glob.set_defaults(func=glob)
494
495
496 args = parser.parse_args()
497
498 if args.debug:
499 logger.setLevel(logging.DEBUG)
500
501 if not args.pkgdata_dir:
502 import scriptpath
503 bitbakepath = scriptpath.add_bitbake_lib_path()
504 if not bitbakepath:
505 logger.error("Unable to find bitbake by searching parent directory of this script or PATH")
506 sys.exit(1)
507 logger.debug('Found bitbake path: %s' % bitbakepath)
508 tinfoil = tinfoil_init()
509 args.pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR', True)
510 logger.debug('Value of PKGDATA_DIR is "%s"' % args.pkgdata_dir)
511 if not args.pkgdata_dir:
512 logger.error('Unable to determine pkgdata directory from PKGDATA_DIR')
513 sys.exit(1)
514
515 if not os.path.exists(args.pkgdata_dir):
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500516 logger.error('Unable to find pkgdata directory %s' % args.pkgdata_dir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500517 sys.exit(1)
518
519 ret = args.func(args)
520
521 return ret
522
523
524if __name__ == "__main__":
525 main()