blob: 796a98f05311b662a1838e60d3df5439aedee84c [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# tinfoil: a simple wrapper around cooker for bitbake-based command-line utilities
2#
Brad Bishop6e60e8b2018-02-01 10:27:11 -05003# Copyright (C) 2012-2017 Intel Corporation
Patrick Williamsc124f4f2015-09-15 14:41:29 -05004# Copyright (C) 2011 Mentor Graphics Corporation
Brad Bishopd7bf8c12018-02-25 22:55:05 -05005# Copyright (C) 2006-2012 Richard Purdie
Patrick Williamsc124f4f2015-09-15 14:41:29 -05006#
Brad Bishopc342db32019-05-15 21:57:59 -04007# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -05008#
Patrick Williamsc124f4f2015-09-15 14:41:29 -05009
10import logging
Patrick Williamsc124f4f2015-09-15 14:41:29 -050011import os
12import sys
Brad Bishop6e60e8b2018-02-01 10:27:11 -050013import atexit
14import re
15from collections import OrderedDict, defaultdict
Andrew Geissler82c905d2020-04-13 13:39:40 -050016from functools import partial
Patrick Williamsc124f4f2015-09-15 14:41:29 -050017
18import bb.cache
19import bb.cooker
20import bb.providers
Brad Bishop6e60e8b2018-02-01 10:27:11 -050021import bb.taskdata
Patrick Williamsc124f4f2015-09-15 14:41:29 -050022import bb.utils
Brad Bishop6e60e8b2018-02-01 10:27:11 -050023import bb.command
24import bb.remotedata
Andrew Geissler82c905d2020-04-13 13:39:40 -050025from bb.main import setup_bitbake, BitBakeConfigParameters
Patrick Williamsc124f4f2015-09-15 14:41:29 -050026import bb.fetch2
27
Brad Bishop6e60e8b2018-02-01 10:27:11 -050028
29# We need this in order to shut down the connection to the bitbake server,
30# otherwise the process will never properly exit
31_server_connections = []
32def _terminate_connections():
33 for connection in _server_connections:
34 connection.terminate()
35atexit.register(_terminate_connections)
36
37class TinfoilUIException(Exception):
38 """Exception raised when the UI returns non-zero from its main function"""
39 def __init__(self, returncode):
40 self.returncode = returncode
41 def __repr__(self):
42 return 'UI module main returned %d' % self.returncode
43
44class TinfoilCommandFailed(Exception):
45 """Exception raised when run_command fails"""
46
Andrew Geissler82c905d2020-04-13 13:39:40 -050047class TinfoilDataStoreConnectorVarHistory:
48 def __init__(self, tinfoil, dsindex):
49 self.tinfoil = tinfoil
50 self.dsindex = dsindex
51
52 def remoteCommand(self, cmd, *args, **kwargs):
53 return self.tinfoil.run_command('dataStoreConnectorVarHistCmd', self.dsindex, cmd, args, kwargs)
54
55 def __getattr__(self, name):
56 if not hasattr(bb.data_smart.VariableHistory, name):
57 raise AttributeError("VariableHistory has no such method %s" % name)
58
59 newfunc = partial(self.remoteCommand, name)
60 setattr(self, name, newfunc)
61 return newfunc
62
63class TinfoilDataStoreConnectorIncHistory:
64 def __init__(self, tinfoil, dsindex):
65 self.tinfoil = tinfoil
66 self.dsindex = dsindex
67
68 def remoteCommand(self, cmd, *args, **kwargs):
69 return self.tinfoil.run_command('dataStoreConnectorIncHistCmd', self.dsindex, cmd, args, kwargs)
70
71 def __getattr__(self, name):
72 if not hasattr(bb.data_smart.IncludeHistory, name):
73 raise AttributeError("IncludeHistory has no such method %s" % name)
74
75 newfunc = partial(self.remoteCommand, name)
76 setattr(self, name, newfunc)
77 return newfunc
78
Brad Bishop6e60e8b2018-02-01 10:27:11 -050079class TinfoilDataStoreConnector:
Andrew Geissler82c905d2020-04-13 13:39:40 -050080 """
81 Connector object used to enable access to datastore objects via tinfoil
82 Method calls are transmitted to the remote datastore for processing, if a datastore is
83 returned we return a connector object for the new store
84 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -050085
86 def __init__(self, tinfoil, dsindex):
87 self.tinfoil = tinfoil
88 self.dsindex = dsindex
Andrew Geissler82c905d2020-04-13 13:39:40 -050089 self.varhistory = TinfoilDataStoreConnectorVarHistory(tinfoil, dsindex)
90 self.inchistory = TinfoilDataStoreConnectorIncHistory(tinfoil, dsindex)
91
92 def remoteCommand(self, cmd, *args, **kwargs):
93 ret = self.tinfoil.run_command('dataStoreConnectorCmd', self.dsindex, cmd, args, kwargs)
94 if isinstance(ret, bb.command.DataStoreConnectionHandle):
95 return TinfoilDataStoreConnector(self.tinfoil, ret.dsindex)
Brad Bishop6e60e8b2018-02-01 10:27:11 -050096 return ret
Andrew Geissler82c905d2020-04-13 13:39:40 -050097
98 def __getattr__(self, name):
99 if not hasattr(bb.data._dict_type, name):
100 raise AttributeError("Data store has no such method %s" % name)
101
102 newfunc = partial(self.remoteCommand, name)
103 setattr(self, name, newfunc)
104 return newfunc
105
106 def __iter__(self):
107 keys = self.tinfoil.run_command('dataStoreConnectorCmd', self.dsindex, "keys", [], {})
108 for k in keys:
109 yield k
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500110
111class TinfoilCookerAdapter:
112 """
113 Provide an adapter for existing code that expects to access a cooker object via Tinfoil,
114 since now Tinfoil is on the client side it no longer has direct access.
115 """
116
117 class TinfoilCookerCollectionAdapter:
118 """ cooker.collection adapter """
Andrew Geissler5a43b432020-06-13 10:46:56 -0500119 def __init__(self, tinfoil, mc=''):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500120 self.tinfoil = tinfoil
Andrew Geissler5a43b432020-06-13 10:46:56 -0500121 self.mc = mc
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500122 def get_file_appends(self, fn):
Andrew Geissler5a43b432020-06-13 10:46:56 -0500123 return self.tinfoil.get_file_appends(fn, self.mc)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500124 def __getattr__(self, name):
125 if name == 'overlayed':
Andrew Geissler5a43b432020-06-13 10:46:56 -0500126 return self.tinfoil.get_overlayed_recipes(self.mc)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500127 elif name == 'bbappends':
Andrew Geissler5a43b432020-06-13 10:46:56 -0500128 return self.tinfoil.run_command('getAllAppends', self.mc)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500129 else:
130 raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
131
132 class TinfoilRecipeCacheAdapter:
133 """ cooker.recipecache adapter """
Andrew Geissler82c905d2020-04-13 13:39:40 -0500134 def __init__(self, tinfoil, mc=''):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500135 self.tinfoil = tinfoil
Andrew Geissler82c905d2020-04-13 13:39:40 -0500136 self.mc = mc
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500137 self._cache = {}
138
139 def get_pkg_pn_fn(self):
Andrew Geissler82c905d2020-04-13 13:39:40 -0500140 pkg_pn = defaultdict(list, self.tinfoil.run_command('getRecipes', self.mc) or [])
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500141 pkg_fn = {}
142 for pn, fnlist in pkg_pn.items():
143 for fn in fnlist:
144 pkg_fn[fn] = pn
145 self._cache['pkg_pn'] = pkg_pn
146 self._cache['pkg_fn'] = pkg_fn
147
148 def __getattr__(self, name):
149 # Grab these only when they are requested since they aren't always used
150 if name in self._cache:
151 return self._cache[name]
152 elif name == 'pkg_pn':
153 self.get_pkg_pn_fn()
154 return self._cache[name]
155 elif name == 'pkg_fn':
156 self.get_pkg_pn_fn()
157 return self._cache[name]
158 elif name == 'deps':
Andrew Geissler82c905d2020-04-13 13:39:40 -0500159 attrvalue = defaultdict(list, self.tinfoil.run_command('getRecipeDepends', self.mc) or [])
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500160 elif name == 'rundeps':
Andrew Geissler82c905d2020-04-13 13:39:40 -0500161 attrvalue = defaultdict(lambda: defaultdict(list), self.tinfoil.run_command('getRuntimeDepends', self.mc) or [])
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500162 elif name == 'runrecs':
Andrew Geissler82c905d2020-04-13 13:39:40 -0500163 attrvalue = defaultdict(lambda: defaultdict(list), self.tinfoil.run_command('getRuntimeRecommends', self.mc) or [])
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500164 elif name == 'pkg_pepvpr':
Andrew Geissler82c905d2020-04-13 13:39:40 -0500165 attrvalue = self.tinfoil.run_command('getRecipeVersions', self.mc) or {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500166 elif name == 'inherits':
Andrew Geissler82c905d2020-04-13 13:39:40 -0500167 attrvalue = self.tinfoil.run_command('getRecipeInherits', self.mc) or {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500168 elif name == 'bbfile_priority':
Andrew Geissler82c905d2020-04-13 13:39:40 -0500169 attrvalue = self.tinfoil.run_command('getBbFilePriority', self.mc) or {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500170 elif name == 'pkg_dp':
Andrew Geissler82c905d2020-04-13 13:39:40 -0500171 attrvalue = self.tinfoil.run_command('getDefaultPreference', self.mc) or {}
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500172 elif name == 'fn_provides':
Andrew Geissler82c905d2020-04-13 13:39:40 -0500173 attrvalue = self.tinfoil.run_command('getRecipeProvides', self.mc) or {}
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500174 elif name == 'packages':
Andrew Geissler82c905d2020-04-13 13:39:40 -0500175 attrvalue = self.tinfoil.run_command('getRecipePackages', self.mc) or {}
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500176 elif name == 'packages_dynamic':
Andrew Geissler82c905d2020-04-13 13:39:40 -0500177 attrvalue = self.tinfoil.run_command('getRecipePackagesDynamic', self.mc) or {}
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500178 elif name == 'rproviders':
Andrew Geissler82c905d2020-04-13 13:39:40 -0500179 attrvalue = self.tinfoil.run_command('getRProviders', self.mc) or {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500180 else:
181 raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
182
183 self._cache[name] = attrvalue
184 return attrvalue
185
186 def __init__(self, tinfoil):
187 self.tinfoil = tinfoil
Andrew Geissler5a43b432020-06-13 10:46:56 -0500188 self.multiconfigs = [''] + (tinfoil.config_data.getVar('BBMULTICONFIG') or '').split()
189 self.collections = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500190 self.recipecaches = {}
Andrew Geissler5a43b432020-06-13 10:46:56 -0500191 for mc in self.multiconfigs:
192 self.collections[mc] = self.TinfoilCookerCollectionAdapter(tinfoil, mc)
Andrew Geissler82c905d2020-04-13 13:39:40 -0500193 self.recipecaches[mc] = self.TinfoilRecipeCacheAdapter(tinfoil, mc)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500194 self._cache = {}
195 def __getattr__(self, name):
196 # Grab these only when they are requested since they aren't always used
197 if name in self._cache:
198 return self._cache[name]
199 elif name == 'skiplist':
200 attrvalue = self.tinfoil.get_skipped_recipes()
201 elif name == 'bbfile_config_priorities':
202 ret = self.tinfoil.run_command('getLayerPriorities')
203 bbfile_config_priorities = []
204 for collection, pattern, regex, pri in ret:
205 bbfile_config_priorities.append((collection, pattern, re.compile(regex), pri))
206
207 attrvalue = bbfile_config_priorities
208 else:
209 raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
210
211 self._cache[name] = attrvalue
212 return attrvalue
213
214 def findBestProvider(self, pn):
215 return self.tinfoil.find_best_provider(pn)
216
217
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500218class TinfoilRecipeInfo:
219 """
220 Provides a convenient representation of the cached information for a single recipe.
221 Some attributes are set on construction, others are read on-demand (which internally
222 may result in a remote procedure call to the bitbake server the first time).
223 Note that only information which is cached is available through this object - if
224 you need other variable values you will need to parse the recipe using
225 Tinfoil.parse_recipe().
226 """
227 def __init__(self, recipecache, d, pn, fn, fns):
228 self._recipecache = recipecache
229 self._d = d
230 self.pn = pn
231 self.fn = fn
232 self.fns = fns
233 self.inherit_files = recipecache.inherits[fn]
234 self.depends = recipecache.deps[fn]
235 (self.pe, self.pv, self.pr) = recipecache.pkg_pepvpr[fn]
236 self._cached_packages = None
237 self._cached_rprovides = None
238 self._cached_packages_dynamic = None
239
240 def __getattr__(self, name):
241 if name == 'alternates':
242 return [x for x in self.fns if x != self.fn]
243 elif name == 'rdepends':
244 return self._recipecache.rundeps[self.fn]
245 elif name == 'rrecommends':
246 return self._recipecache.runrecs[self.fn]
247 elif name == 'provides':
248 return self._recipecache.fn_provides[self.fn]
249 elif name == 'packages':
250 if self._cached_packages is None:
251 self._cached_packages = []
252 for pkg, fns in self._recipecache.packages.items():
253 if self.fn in fns:
254 self._cached_packages.append(pkg)
255 return self._cached_packages
256 elif name == 'packages_dynamic':
257 if self._cached_packages_dynamic is None:
258 self._cached_packages_dynamic = []
259 for pkg, fns in self._recipecache.packages_dynamic.items():
260 if self.fn in fns:
261 self._cached_packages_dynamic.append(pkg)
262 return self._cached_packages_dynamic
263 elif name == 'rprovides':
264 if self._cached_rprovides is None:
265 self._cached_rprovides = []
266 for pkg, fns in self._recipecache.rproviders.items():
267 if self.fn in fns:
268 self._cached_rprovides.append(pkg)
269 return self._cached_rprovides
270 else:
271 raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
272 def inherits(self, only_recipe=False):
273 """
274 Get the inherited classes for a recipe. Returns the class names only.
275 Parameters:
276 only_recipe: True to return only the classes inherited by the recipe
277 itself, False to return all classes inherited within
278 the context for the recipe (which includes globally
279 inherited classes).
280 """
281 if only_recipe:
282 global_inherit = [x for x in (self._d.getVar('BBINCLUDED') or '').split() if x.endswith('.bbclass')]
283 else:
284 global_inherit = []
285 for clsfile in self.inherit_files:
286 if only_recipe and clsfile in global_inherit:
287 continue
288 clsname = os.path.splitext(os.path.basename(clsfile))[0]
289 yield clsname
290 def __str__(self):
291 return '%s' % self.pn
292
293
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500294class Tinfoil:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500295 """
296 Tinfoil - an API for scripts and utilities to query
297 BitBake internals and perform build operations.
298 """
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500299
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500300 def __init__(self, output=sys.stdout, tracking=False, setup_logging=True):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500301 """
302 Create a new tinfoil object.
303 Parameters:
304 output: specifies where console output should be sent. Defaults
305 to sys.stdout.
306 tracking: True to enable variable history tracking, False to
307 disable it (default). Enabling this has a minor
308 performance impact so typically it isn't enabled
309 unless you need to query variable history.
310 setup_logging: True to setup a logger so that things like
311 bb.warn() will work immediately and timeout warnings
312 are visible; False to let BitBake do this itself.
313 """
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500314 self.logger = logging.getLogger('BitBake')
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500315 self.config_data = None
316 self.cooker = None
317 self.tracking = tracking
318 self.ui_module = None
319 self.server_connection = None
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500320 self.recipes_parsed = False
321 self.quiet = 0
322 self.oldhandlers = self.logger.handlers[:]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500323 if setup_logging:
324 # This is the *client-side* logger, nothing to do with
325 # logging messages from the server
326 bb.msg.logger_create('BitBake', output)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500327 self.localhandlers = []
328 for handler in self.logger.handlers:
329 if handler not in self.oldhandlers:
330 self.localhandlers.append(handler)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500331
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600332 def __enter__(self):
333 return self
334
335 def __exit__(self, type, value, traceback):
336 self.shutdown()
337
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500338 def prepare(self, config_only=False, config_params=None, quiet=0, extra_features=None):
339 """
340 Prepares the underlying BitBake system to be used via tinfoil.
341 This function must be called prior to calling any of the other
342 functions in the API.
343 NOTE: if you call prepare() you must absolutely call shutdown()
344 before your code terminates. You can use a "with" block to ensure
345 this happens e.g.
346
347 with bb.tinfoil.Tinfoil() as tinfoil:
348 tinfoil.prepare()
349 ...
350
351 Parameters:
352 config_only: True to read only the configuration and not load
353 the cache / parse recipes. This is useful if you just
354 want to query the value of a variable at the global
355 level or you want to do anything else that doesn't
356 involve knowing anything about the recipes in the
357 current configuration. False loads the cache / parses
358 recipes.
359 config_params: optionally specify your own configuration
360 parameters. If not specified an instance of
361 TinfoilConfigParameters will be created internally.
362 quiet: quiet level controlling console output - equivalent
363 to bitbake's -q/--quiet option. Default of 0 gives
364 the same output level as normal bitbake execution.
365 extra_features: extra features to be added to the feature
366 set requested from the server. See
367 CookerFeatures._feature_list for possible
368 features.
369 """
370 self.quiet = quiet
371
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500372 if self.tracking:
373 extrafeatures = [bb.cooker.CookerFeatures.BASEDATASTORE_TRACKING]
374 else:
375 extrafeatures = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500376
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500377 if extra_features:
378 extrafeatures += extra_features
379
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500380 if not config_params:
381 config_params = TinfoilConfigParameters(config_only=config_only, quiet=quiet)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500382
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500383 if not config_only:
384 # Disable local loggers because the UI module is going to set up its own
385 for handler in self.localhandlers:
386 self.logger.handlers.remove(handler)
387 self.localhandlers = []
388
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500389 self.server_connection, ui_module = setup_bitbake(config_params, extrafeatures)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500390
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500391 self.ui_module = ui_module
392
393 # Ensure the path to bitbake's bin directory is in PATH so that things like
394 # bitbake-worker can be run (usually this is the case, but it doesn't have to be)
395 path = os.getenv('PATH').split(':')
396 bitbakebinpath = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', 'bin'))
397 for entry in path:
398 if entry.endswith(os.sep):
399 entry = entry[:-1]
400 if os.path.abspath(entry) == bitbakebinpath:
401 break
402 else:
403 path.insert(0, bitbakebinpath)
404 os.environ['PATH'] = ':'.join(path)
405
406 if self.server_connection:
407 _server_connections.append(self.server_connection)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500408 if config_only:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500409 config_params.updateToServer(self.server_connection.connection, os.environ.copy())
410 self.run_command('parseConfiguration')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500411 else:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500412 self.run_actions(config_params)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500413 self.recipes_parsed = True
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500414
Andrew Geissler82c905d2020-04-13 13:39:40 -0500415 self.config_data = TinfoilDataStoreConnector(self, 0)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500416 self.cooker = TinfoilCookerAdapter(self)
417 self.cooker_data = self.cooker.recipecaches['']
418 else:
419 raise Exception('Failed to start bitbake server')
420
421 def run_actions(self, config_params):
422 """
423 Run the actions specified in config_params through the UI.
424 """
425 ret = self.ui_module.main(self.server_connection.connection, self.server_connection.events, config_params)
426 if ret:
427 raise TinfoilUIException(ret)
428
429 def parseRecipes(self):
430 """
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500431 Legacy function - use parse_recipes() instead.
432 """
433 self.parse_recipes()
434
435 def parse_recipes(self):
436 """
437 Load information on all recipes. Normally you should specify
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500438 config_only=False when calling prepare() instead of using this
439 function; this function is designed for situations where you need
440 to initialise Tinfoil and use it with config_only=True first and
441 then conditionally call this function to parse recipes later.
442 """
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500443 config_params = TinfoilConfigParameters(config_only=False, quiet=self.quiet)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500444 self.run_actions(config_params)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500445 self.recipes_parsed = True
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500446
447 def run_command(self, command, *params):
448 """
449 Run a command on the server (as implemented in bb.command).
450 Note that there are two types of command - synchronous and
451 asynchronous; in order to receive the results of asynchronous
452 commands you will need to set an appropriate event mask
453 using set_event_mask() and listen for the result using
454 wait_event() - with the correct event mask you'll at least get
455 bb.command.CommandCompleted and possibly other events before
456 that depending on the command.
457 """
458 if not self.server_connection:
459 raise Exception('Not connected to server (did you call .prepare()?)')
460
461 commandline = [command]
462 if params:
463 commandline.extend(params)
Andrew Geisslerf0343792020-11-18 10:42:21 -0600464 try:
465 result = self.server_connection.connection.runCommand(commandline)
466 finally:
467 while True:
468 event = self.wait_event()
469 if not event:
470 break
471 if isinstance(event, logging.LogRecord):
472 if event.taskpid == 0 or event.levelno > logging.INFO:
473 self.logger.handle(event)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500474 if result[1]:
475 raise TinfoilCommandFailed(result[1])
476 return result[0]
477
478 def set_event_mask(self, eventlist):
479 """Set the event mask which will be applied within wait_event()"""
480 if not self.server_connection:
481 raise Exception('Not connected to server (did you call .prepare()?)')
482 llevel, debug_domains = bb.msg.constructLogOptions()
483 ret = self.run_command('setEventMask', self.server_connection.connection.getEventHandle(), llevel, debug_domains, eventlist)
484 if not ret:
485 raise Exception('setEventMask failed')
486
487 def wait_event(self, timeout=0):
488 """
489 Wait for an event from the server for the specified time.
490 A timeout of 0 means don't wait if there are no events in the queue.
491 Returns the next event in the queue or None if the timeout was
492 reached. Note that in order to recieve any events you will
493 first need to set the internal event mask using set_event_mask()
494 (otherwise whatever event mask the UI set up will be in effect).
495 """
496 if not self.server_connection:
497 raise Exception('Not connected to server (did you call .prepare()?)')
498 return self.server_connection.events.waitEvent(timeout)
499
Andrew Geissler5a43b432020-06-13 10:46:56 -0500500 def get_overlayed_recipes(self, mc=''):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500501 """
502 Find recipes which are overlayed (i.e. where recipes exist in multiple layers)
503 """
Andrew Geissler5a43b432020-06-13 10:46:56 -0500504 return defaultdict(list, self.run_command('getOverlayedRecipes', mc))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500505
506 def get_skipped_recipes(self):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500507 """
508 Find recipes which were skipped (i.e. SkipRecipe was raised
509 during parsing).
510 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500511 return OrderedDict(self.run_command('getSkippedRecipes'))
512
Andrew Geissler82c905d2020-04-13 13:39:40 -0500513 def get_all_providers(self, mc=''):
514 return defaultdict(list, self.run_command('allProviders', mc))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500515
Andrew Geissler82c905d2020-04-13 13:39:40 -0500516 def find_providers(self, mc=''):
517 return self.run_command('findProviders', mc)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500518
519 def find_best_provider(self, pn):
520 return self.run_command('findBestProvider', pn)
521
522 def get_runtime_providers(self, rdep):
523 return self.run_command('getRuntimeProviders', rdep)
524
525 def get_recipe_file(self, pn):
526 """
527 Get the file name for the specified recipe/target. Raises
528 bb.providers.NoProvider if there is no match or the recipe was
529 skipped.
530 """
531 best = self.find_best_provider(pn)
532 if not best or (len(best) > 3 and not best[3]):
533 skiplist = self.get_skipped_recipes()
534 taskdata = bb.taskdata.TaskData(None, skiplist=skiplist)
535 skipreasons = taskdata.get_reasons(pn)
536 if skipreasons:
537 raise bb.providers.NoProvider('%s is unavailable:\n %s' % (pn, ' \n'.join(skipreasons)))
538 else:
539 raise bb.providers.NoProvider('Unable to find any recipe file matching "%s"' % pn)
540 return best[3]
541
Andrew Geissler5a43b432020-06-13 10:46:56 -0500542 def get_file_appends(self, fn, mc=''):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500543 """
544 Find the bbappends for a recipe file
545 """
Andrew Geissler5a43b432020-06-13 10:46:56 -0500546 return self.run_command('getFileAppends', fn, mc)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500547
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500548 def all_recipes(self, mc='', sort=True):
549 """
550 Enable iterating over all recipes in the current configuration.
551 Returns an iterator over TinfoilRecipeInfo objects created on demand.
552 Parameters:
553 mc: The multiconfig, default of '' uses the main configuration.
554 sort: True to sort recipes alphabetically (default), False otherwise
555 """
556 recipecache = self.cooker.recipecaches[mc]
557 if sort:
558 recipes = sorted(recipecache.pkg_pn.items())
559 else:
560 recipes = recipecache.pkg_pn.items()
561 for pn, fns in recipes:
562 prov = self.find_best_provider(pn)
563 recipe = TinfoilRecipeInfo(recipecache,
564 self.config_data,
565 pn=pn,
566 fn=prov[3],
567 fns=fns)
568 yield recipe
569
570 def all_recipe_files(self, mc='', variants=True, preferred_only=False):
571 """
572 Enable iterating over all recipe files in the current configuration.
573 Returns an iterator over file paths.
574 Parameters:
575 mc: The multiconfig, default of '' uses the main configuration.
576 variants: True to include variants of recipes created through
577 BBCLASSEXTEND (default) or False to exclude them
578 preferred_only: True to include only the preferred recipe where
579 multiple exist providing the same PN, False to list
580 all recipes
581 """
582 recipecache = self.cooker.recipecaches[mc]
583 if preferred_only:
584 files = []
585 for pn in recipecache.pkg_pn.keys():
586 prov = self.find_best_provider(pn)
587 files.append(prov[3])
588 else:
589 files = recipecache.pkg_fn.keys()
590 for fn in sorted(files):
591 if not variants and fn.startswith('virtual:'):
592 continue
593 yield fn
594
595
596 def get_recipe_info(self, pn, mc=''):
597 """
598 Get information on a specific recipe in the current configuration by name (PN).
599 Returns a TinfoilRecipeInfo object created on demand.
600 Parameters:
601 mc: The multiconfig, default of '' uses the main configuration.
602 """
603 recipecache = self.cooker.recipecaches[mc]
604 prov = self.find_best_provider(pn)
605 fn = prov[3]
Brad Bishop316dfdd2018-06-25 12:45:53 -0400606 if fn:
607 actual_pn = recipecache.pkg_fn[fn]
608 recipe = TinfoilRecipeInfo(recipecache,
609 self.config_data,
610 pn=actual_pn,
611 fn=fn,
612 fns=recipecache.pkg_pn[actual_pn])
613 return recipe
614 else:
615 return None
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500616
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500617 def parse_recipe(self, pn):
618 """
619 Parse the specified recipe and return a datastore object
620 representing the environment for the recipe.
621 """
622 fn = self.get_recipe_file(pn)
623 return self.parse_recipe_file(fn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500624
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600625 def parse_recipe_file(self, fn, appends=True, appendlist=None, config_data=None):
626 """
627 Parse the specified recipe file (with or without bbappends)
628 and return a datastore object representing the environment
629 for the recipe.
630 Parameters:
631 fn: recipe file to parse - can be a file path or virtual
632 specification
633 appends: True to apply bbappends, False otherwise
634 appendlist: optional list of bbappend files to apply, if you
635 want to filter them
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600636 """
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500637 if self.tracking:
638 # Enable history tracking just for the parse operation
639 self.run_command('enableDataTracking')
640 try:
641 if appends and appendlist == []:
642 appends = False
643 if config_data:
Andrew Geissler82c905d2020-04-13 13:39:40 -0500644 config_data = bb.data.createCopy(config_data)
645 dscon = self.run_command('parseRecipeFile', fn, appends, appendlist, config_data.dsindex)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500646 else:
647 dscon = self.run_command('parseRecipeFile', fn, appends, appendlist)
648 if dscon:
649 return self._reconvert_type(dscon, 'DataStoreConnectionHandle')
650 else:
651 return None
652 finally:
653 if self.tracking:
654 self.run_command('disableDataTracking')
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500655
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500656 def build_file(self, buildfile, task, internal=True):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500657 """
658 Runs the specified task for just a single recipe (i.e. no dependencies).
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500659 This is equivalent to bitbake -b, except with the default internal=True
660 no warning about dependencies will be produced, normal info messages
661 from the runqueue will be silenced and BuildInit, BuildStarted and
662 BuildCompleted events will not be fired.
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500663 """
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500664 return self.run_command('buildFile', buildfile, task, internal)
665
666 def build_targets(self, targets, task=None, handle_events=True, extra_events=None, event_callback=None):
667 """
668 Builds the specified targets. This is equivalent to a normal invocation
669 of bitbake. Has built-in event handling which is enabled by default and
670 can be extended if needed.
671 Parameters:
672 targets:
673 One or more targets to build. Can be a list or a
674 space-separated string.
675 task:
676 The task to run; if None then the value of BB_DEFAULT_TASK
677 will be used. Default None.
678 handle_events:
679 True to handle events in a similar way to normal bitbake
680 invocation with knotty; False to return immediately (on the
681 assumption that the caller will handle the events instead).
682 Default True.
683 extra_events:
684 An optional list of events to add to the event mask (if
685 handle_events=True). If you add events here you also need
686 to specify a callback function in event_callback that will
687 handle the additional events. Default None.
688 event_callback:
689 An optional function taking a single parameter which
690 will be called first upon receiving any event (if
691 handle_events=True) so that the caller can override or
692 extend the event handling. Default None.
693 """
694 if isinstance(targets, str):
695 targets = targets.split()
696 if not task:
697 task = self.config_data.getVar('BB_DEFAULT_TASK')
698
699 if handle_events:
700 # A reasonable set of default events matching up with those we handle below
701 eventmask = [
702 'bb.event.BuildStarted',
703 'bb.event.BuildCompleted',
704 'logging.LogRecord',
705 'bb.event.NoProvider',
706 'bb.command.CommandCompleted',
707 'bb.command.CommandFailed',
708 'bb.build.TaskStarted',
709 'bb.build.TaskFailed',
710 'bb.build.TaskSucceeded',
711 'bb.build.TaskFailedSilent',
712 'bb.build.TaskProgress',
713 'bb.runqueue.runQueueTaskStarted',
714 'bb.runqueue.sceneQueueTaskStarted',
715 'bb.event.ProcessStarted',
716 'bb.event.ProcessProgress',
717 'bb.event.ProcessFinished',
718 ]
719 if extra_events:
720 eventmask.extend(extra_events)
721 ret = self.set_event_mask(eventmask)
722
723 includelogs = self.config_data.getVar('BBINCLUDELOGS')
724 loglines = self.config_data.getVar('BBINCLUDELOGS_LINES')
725
726 ret = self.run_command('buildTargets', targets, task)
727 if handle_events:
728 result = False
729 # Borrowed from knotty, instead somewhat hackily we use the helper
730 # as the object to store "shutdown" on
731 helper = bb.ui.uihelper.BBUIHelper()
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500732 helper.shutdown = 0
733 parseprogress = None
Andrew Geissler82c905d2020-04-13 13:39:40 -0500734 termfilter = bb.ui.knotty.TerminalFilter(helper, helper, self.logger.handlers, quiet=self.quiet)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500735 try:
736 while True:
737 try:
738 event = self.wait_event(0.25)
739 if event:
740 if event_callback and event_callback(event):
741 continue
742 if helper.eventHandler(event):
743 if isinstance(event, bb.build.TaskFailedSilent):
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500744 self.logger.warning("Logfile for failed setscene task is %s" % event.logfile)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500745 elif isinstance(event, bb.build.TaskFailed):
746 bb.ui.knotty.print_event_log(event, includelogs, loglines, termfilter)
747 continue
748 if isinstance(event, bb.event.ProcessStarted):
749 if self.quiet > 1:
750 continue
751 parseprogress = bb.ui.knotty.new_progress(event.processname, event.total)
752 parseprogress.start(False)
753 continue
754 if isinstance(event, bb.event.ProcessProgress):
755 if self.quiet > 1:
756 continue
757 if parseprogress:
758 parseprogress.update(event.progress)
759 else:
760 bb.warn("Got ProcessProgress event for someting that never started?")
761 continue
762 if isinstance(event, bb.event.ProcessFinished):
763 if self.quiet > 1:
764 continue
765 if parseprogress:
766 parseprogress.finish()
767 parseprogress = None
768 continue
769 if isinstance(event, bb.command.CommandCompleted):
770 result = True
771 break
772 if isinstance(event, bb.command.CommandFailed):
773 self.logger.error(str(event))
774 result = False
775 break
776 if isinstance(event, logging.LogRecord):
777 if event.taskpid == 0 or event.levelno > logging.INFO:
778 self.logger.handle(event)
779 continue
780 if isinstance(event, bb.event.NoProvider):
781 self.logger.error(str(event))
782 result = False
783 break
784
785 elif helper.shutdown > 1:
786 break
787 termfilter.updateFooter()
788 except KeyboardInterrupt:
789 termfilter.clearFooter()
790 if helper.shutdown == 1:
791 print("\nSecond Keyboard Interrupt, stopping...\n")
792 ret = self.run_command("stateForceShutdown")
793 if ret and ret[2]:
794 self.logger.error("Unable to cleanly stop: %s" % ret[2])
795 elif helper.shutdown == 0:
796 print("\nKeyboard Interrupt, closing down...\n")
797 interrupted = True
798 ret = self.run_command("stateShutdown")
799 if ret and ret[2]:
800 self.logger.error("Unable to cleanly shutdown: %s" % ret[2])
801 helper.shutdown = helper.shutdown + 1
802 termfilter.clearFooter()
803 finally:
804 termfilter.finish()
805 if helper.failed_tasks:
806 result = False
807 return result
808 else:
809 return ret
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600810
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500811 def shutdown(self):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500812 """
813 Shut down tinfoil. Disconnects from the server and gracefully
814 releases any associated resources. You must call this function if
815 prepare() has been called, or use a with... block when you create
816 the tinfoil object which will ensure that it gets called.
817 """
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500818 try:
819 if self.server_connection:
820 try:
821 self.run_command('clientComplete')
822 finally:
823 _server_connections.remove(self.server_connection)
824 bb.event.ui_queue = []
825 self.server_connection.terminate()
826 self.server_connection = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500827
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500828 finally:
829 # Restore logging handlers to how it looked when we started
830 if self.oldhandlers:
831 for handler in self.logger.handlers:
832 if handler not in self.oldhandlers:
833 self.logger.handlers.remove(handler)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500834
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500835 def _reconvert_type(self, obj, origtypename):
836 """
837 Convert an object back to the right type, in the case
838 that marshalling has changed it (especially with xmlrpc)
839 """
840 supported_types = {
841 'set': set,
842 'DataStoreConnectionHandle': bb.command.DataStoreConnectionHandle,
843 }
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500844
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500845 origtype = supported_types.get(origtypename, None)
846 if origtype is None:
847 raise Exception('Unsupported type "%s"' % origtypename)
848 if type(obj) == origtype:
849 newobj = obj
850 elif isinstance(obj, dict):
851 # New style class
852 newobj = origtype()
853 for k,v in obj.items():
854 setattr(newobj, k, v)
855 else:
856 # Assume we can coerce the type
857 newobj = origtype(obj)
858
859 if isinstance(newobj, bb.command.DataStoreConnectionHandle):
Andrew Geissler82c905d2020-04-13 13:39:40 -0500860 newobj = TinfoilDataStoreConnector(self, newobj.dsindex)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500861
862 return newobj
863
864
865class TinfoilConfigParameters(BitBakeConfigParameters):
866
867 def __init__(self, config_only, **options):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500868 self.initial_options = options
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500869 # Apply some sane defaults
870 if not 'parse_only' in options:
871 self.initial_options['parse_only'] = not config_only
872 #if not 'status_only' in options:
873 # self.initial_options['status_only'] = config_only
874 if not 'ui' in options:
875 self.initial_options['ui'] = 'knotty'
876 if not 'argv' in options:
877 self.initial_options['argv'] = []
878
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500879 super(TinfoilConfigParameters, self).__init__()
880
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500881 def parseCommandLine(self, argv=None):
882 # We don't want any parameters parsed from the command line
883 opts = super(TinfoilConfigParameters, self).parseCommandLine([])
884 for key, val in self.initial_options.items():
885 setattr(opts[0], key, val)
886 return opts