blob: 2fb1bb7d27ce395e73a1d3b7433fd11067d88e16 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# tinfoil: a simple wrapper around cooker for bitbake-based command-line utilities
2#
Brad Bishop6e60e8b2018-02-01 10:27:11 -05003# Copyright (C) 2012-2017 Intel Corporation
Patrick Williamsc124f4f2015-09-15 14:41:29 -05004# Copyright (C) 2011 Mentor Graphics Corporation
Brad Bishopd7bf8c12018-02-25 22:55:05 -05005# Copyright (C) 2006-2012 Richard Purdie
Patrick Williamsc124f4f2015-09-15 14:41:29 -05006#
Brad Bishopc342db32019-05-15 21:57:59 -04007# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -05008#
Patrick Williamsc124f4f2015-09-15 14:41:29 -05009
10import logging
Patrick Williamsc124f4f2015-09-15 14:41:29 -050011import os
12import sys
Brad Bishop6e60e8b2018-02-01 10:27:11 -050013import atexit
14import re
15from collections import OrderedDict, defaultdict
Andrew Geissler82c905d2020-04-13 13:39:40 -050016from functools import partial
Patrick Williamsc124f4f2015-09-15 14:41:29 -050017
18import bb.cache
19import bb.cooker
20import bb.providers
Brad Bishop6e60e8b2018-02-01 10:27:11 -050021import bb.taskdata
Patrick Williamsc124f4f2015-09-15 14:41:29 -050022import bb.utils
Brad Bishop6e60e8b2018-02-01 10:27:11 -050023import bb.command
24import bb.remotedata
Andrew Geissler82c905d2020-04-13 13:39:40 -050025from bb.main import setup_bitbake, BitBakeConfigParameters
Patrick Williamsc124f4f2015-09-15 14:41:29 -050026import bb.fetch2
27
Brad Bishop6e60e8b2018-02-01 10:27:11 -050028
29# We need this in order to shut down the connection to the bitbake server,
30# otherwise the process will never properly exit
31_server_connections = []
32def _terminate_connections():
33 for connection in _server_connections:
34 connection.terminate()
35atexit.register(_terminate_connections)
36
37class TinfoilUIException(Exception):
38 """Exception raised when the UI returns non-zero from its main function"""
39 def __init__(self, returncode):
40 self.returncode = returncode
41 def __repr__(self):
42 return 'UI module main returned %d' % self.returncode
43
44class TinfoilCommandFailed(Exception):
45 """Exception raised when run_command fails"""
46
Andrew Geissler82c905d2020-04-13 13:39:40 -050047class TinfoilDataStoreConnectorVarHistory:
48 def __init__(self, tinfoil, dsindex):
49 self.tinfoil = tinfoil
50 self.dsindex = dsindex
51
52 def remoteCommand(self, cmd, *args, **kwargs):
53 return self.tinfoil.run_command('dataStoreConnectorVarHistCmd', self.dsindex, cmd, args, kwargs)
54
55 def __getattr__(self, name):
56 if not hasattr(bb.data_smart.VariableHistory, name):
57 raise AttributeError("VariableHistory has no such method %s" % name)
58
59 newfunc = partial(self.remoteCommand, name)
60 setattr(self, name, newfunc)
61 return newfunc
62
63class TinfoilDataStoreConnectorIncHistory:
64 def __init__(self, tinfoil, dsindex):
65 self.tinfoil = tinfoil
66 self.dsindex = dsindex
67
68 def remoteCommand(self, cmd, *args, **kwargs):
69 return self.tinfoil.run_command('dataStoreConnectorIncHistCmd', self.dsindex, cmd, args, kwargs)
70
71 def __getattr__(self, name):
72 if not hasattr(bb.data_smart.IncludeHistory, name):
73 raise AttributeError("IncludeHistory has no such method %s" % name)
74
75 newfunc = partial(self.remoteCommand, name)
76 setattr(self, name, newfunc)
77 return newfunc
78
Brad Bishop6e60e8b2018-02-01 10:27:11 -050079class TinfoilDataStoreConnector:
Andrew Geissler82c905d2020-04-13 13:39:40 -050080 """
81 Connector object used to enable access to datastore objects via tinfoil
82 Method calls are transmitted to the remote datastore for processing, if a datastore is
83 returned we return a connector object for the new store
84 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -050085
86 def __init__(self, tinfoil, dsindex):
87 self.tinfoil = tinfoil
88 self.dsindex = dsindex
Andrew Geissler82c905d2020-04-13 13:39:40 -050089 self.varhistory = TinfoilDataStoreConnectorVarHistory(tinfoil, dsindex)
90 self.inchistory = TinfoilDataStoreConnectorIncHistory(tinfoil, dsindex)
91
92 def remoteCommand(self, cmd, *args, **kwargs):
93 ret = self.tinfoil.run_command('dataStoreConnectorCmd', self.dsindex, cmd, args, kwargs)
94 if isinstance(ret, bb.command.DataStoreConnectionHandle):
95 return TinfoilDataStoreConnector(self.tinfoil, ret.dsindex)
Brad Bishop6e60e8b2018-02-01 10:27:11 -050096 return ret
Andrew Geissler82c905d2020-04-13 13:39:40 -050097
98 def __getattr__(self, name):
99 if not hasattr(bb.data._dict_type, name):
100 raise AttributeError("Data store has no such method %s" % name)
101
102 newfunc = partial(self.remoteCommand, name)
103 setattr(self, name, newfunc)
104 return newfunc
105
106 def __iter__(self):
107 keys = self.tinfoil.run_command('dataStoreConnectorCmd', self.dsindex, "keys", [], {})
108 for k in keys:
109 yield k
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500110
111class TinfoilCookerAdapter:
112 """
113 Provide an adapter for existing code that expects to access a cooker object via Tinfoil,
114 since now Tinfoil is on the client side it no longer has direct access.
115 """
116
117 class TinfoilCookerCollectionAdapter:
118 """ cooker.collection adapter """
Andrew Geissler5a43b432020-06-13 10:46:56 -0500119 def __init__(self, tinfoil, mc=''):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500120 self.tinfoil = tinfoil
Andrew Geissler5a43b432020-06-13 10:46:56 -0500121 self.mc = mc
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500122 def get_file_appends(self, fn):
Andrew Geissler5a43b432020-06-13 10:46:56 -0500123 return self.tinfoil.get_file_appends(fn, self.mc)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500124 def __getattr__(self, name):
125 if name == 'overlayed':
Andrew Geissler5a43b432020-06-13 10:46:56 -0500126 return self.tinfoil.get_overlayed_recipes(self.mc)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500127 elif name == 'bbappends':
Andrew Geissler5a43b432020-06-13 10:46:56 -0500128 return self.tinfoil.run_command('getAllAppends', self.mc)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500129 else:
130 raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
131
132 class TinfoilRecipeCacheAdapter:
133 """ cooker.recipecache adapter """
Andrew Geissler82c905d2020-04-13 13:39:40 -0500134 def __init__(self, tinfoil, mc=''):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500135 self.tinfoil = tinfoil
Andrew Geissler82c905d2020-04-13 13:39:40 -0500136 self.mc = mc
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500137 self._cache = {}
138
139 def get_pkg_pn_fn(self):
Andrew Geissler82c905d2020-04-13 13:39:40 -0500140 pkg_pn = defaultdict(list, self.tinfoil.run_command('getRecipes', self.mc) or [])
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500141 pkg_fn = {}
142 for pn, fnlist in pkg_pn.items():
143 for fn in fnlist:
144 pkg_fn[fn] = pn
145 self._cache['pkg_pn'] = pkg_pn
146 self._cache['pkg_fn'] = pkg_fn
147
148 def __getattr__(self, name):
149 # Grab these only when they are requested since they aren't always used
150 if name in self._cache:
151 return self._cache[name]
152 elif name == 'pkg_pn':
153 self.get_pkg_pn_fn()
154 return self._cache[name]
155 elif name == 'pkg_fn':
156 self.get_pkg_pn_fn()
157 return self._cache[name]
158 elif name == 'deps':
Andrew Geissler82c905d2020-04-13 13:39:40 -0500159 attrvalue = defaultdict(list, self.tinfoil.run_command('getRecipeDepends', self.mc) or [])
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500160 elif name == 'rundeps':
Andrew Geissler82c905d2020-04-13 13:39:40 -0500161 attrvalue = defaultdict(lambda: defaultdict(list), self.tinfoil.run_command('getRuntimeDepends', self.mc) or [])
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500162 elif name == 'runrecs':
Andrew Geissler82c905d2020-04-13 13:39:40 -0500163 attrvalue = defaultdict(lambda: defaultdict(list), self.tinfoil.run_command('getRuntimeRecommends', self.mc) or [])
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500164 elif name == 'pkg_pepvpr':
Andrew Geissler82c905d2020-04-13 13:39:40 -0500165 attrvalue = self.tinfoil.run_command('getRecipeVersions', self.mc) or {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500166 elif name == 'inherits':
Andrew Geissler82c905d2020-04-13 13:39:40 -0500167 attrvalue = self.tinfoil.run_command('getRecipeInherits', self.mc) or {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500168 elif name == 'bbfile_priority':
Andrew Geissler82c905d2020-04-13 13:39:40 -0500169 attrvalue = self.tinfoil.run_command('getBbFilePriority', self.mc) or {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500170 elif name == 'pkg_dp':
Andrew Geissler82c905d2020-04-13 13:39:40 -0500171 attrvalue = self.tinfoil.run_command('getDefaultPreference', self.mc) or {}
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500172 elif name == 'fn_provides':
Andrew Geissler82c905d2020-04-13 13:39:40 -0500173 attrvalue = self.tinfoil.run_command('getRecipeProvides', self.mc) or {}
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500174 elif name == 'packages':
Andrew Geissler82c905d2020-04-13 13:39:40 -0500175 attrvalue = self.tinfoil.run_command('getRecipePackages', self.mc) or {}
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500176 elif name == 'packages_dynamic':
Andrew Geissler82c905d2020-04-13 13:39:40 -0500177 attrvalue = self.tinfoil.run_command('getRecipePackagesDynamic', self.mc) or {}
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500178 elif name == 'rproviders':
Andrew Geissler82c905d2020-04-13 13:39:40 -0500179 attrvalue = self.tinfoil.run_command('getRProviders', self.mc) or {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500180 else:
181 raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
182
183 self._cache[name] = attrvalue
184 return attrvalue
185
186 def __init__(self, tinfoil):
187 self.tinfoil = tinfoil
Andrew Geissler5a43b432020-06-13 10:46:56 -0500188 self.multiconfigs = [''] + (tinfoil.config_data.getVar('BBMULTICONFIG') or '').split()
189 self.collections = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500190 self.recipecaches = {}
Andrew Geissler5a43b432020-06-13 10:46:56 -0500191 for mc in self.multiconfigs:
192 self.collections[mc] = self.TinfoilCookerCollectionAdapter(tinfoil, mc)
Andrew Geissler82c905d2020-04-13 13:39:40 -0500193 self.recipecaches[mc] = self.TinfoilRecipeCacheAdapter(tinfoil, mc)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500194 self._cache = {}
195 def __getattr__(self, name):
196 # Grab these only when they are requested since they aren't always used
197 if name in self._cache:
198 return self._cache[name]
199 elif name == 'skiplist':
200 attrvalue = self.tinfoil.get_skipped_recipes()
201 elif name == 'bbfile_config_priorities':
202 ret = self.tinfoil.run_command('getLayerPriorities')
203 bbfile_config_priorities = []
204 for collection, pattern, regex, pri in ret:
205 bbfile_config_priorities.append((collection, pattern, re.compile(regex), pri))
206
207 attrvalue = bbfile_config_priorities
208 else:
209 raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
210
211 self._cache[name] = attrvalue
212 return attrvalue
213
214 def findBestProvider(self, pn):
215 return self.tinfoil.find_best_provider(pn)
216
217
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500218class TinfoilRecipeInfo:
219 """
220 Provides a convenient representation of the cached information for a single recipe.
221 Some attributes are set on construction, others are read on-demand (which internally
222 may result in a remote procedure call to the bitbake server the first time).
223 Note that only information which is cached is available through this object - if
224 you need other variable values you will need to parse the recipe using
225 Tinfoil.parse_recipe().
226 """
227 def __init__(self, recipecache, d, pn, fn, fns):
228 self._recipecache = recipecache
229 self._d = d
230 self.pn = pn
231 self.fn = fn
232 self.fns = fns
233 self.inherit_files = recipecache.inherits[fn]
234 self.depends = recipecache.deps[fn]
235 (self.pe, self.pv, self.pr) = recipecache.pkg_pepvpr[fn]
236 self._cached_packages = None
237 self._cached_rprovides = None
238 self._cached_packages_dynamic = None
239
240 def __getattr__(self, name):
241 if name == 'alternates':
242 return [x for x in self.fns if x != self.fn]
243 elif name == 'rdepends':
244 return self._recipecache.rundeps[self.fn]
245 elif name == 'rrecommends':
246 return self._recipecache.runrecs[self.fn]
247 elif name == 'provides':
248 return self._recipecache.fn_provides[self.fn]
249 elif name == 'packages':
250 if self._cached_packages is None:
251 self._cached_packages = []
252 for pkg, fns in self._recipecache.packages.items():
253 if self.fn in fns:
254 self._cached_packages.append(pkg)
255 return self._cached_packages
256 elif name == 'packages_dynamic':
257 if self._cached_packages_dynamic is None:
258 self._cached_packages_dynamic = []
259 for pkg, fns in self._recipecache.packages_dynamic.items():
260 if self.fn in fns:
261 self._cached_packages_dynamic.append(pkg)
262 return self._cached_packages_dynamic
263 elif name == 'rprovides':
264 if self._cached_rprovides is None:
265 self._cached_rprovides = []
266 for pkg, fns in self._recipecache.rproviders.items():
267 if self.fn in fns:
268 self._cached_rprovides.append(pkg)
269 return self._cached_rprovides
270 else:
271 raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
272 def inherits(self, only_recipe=False):
273 """
274 Get the inherited classes for a recipe. Returns the class names only.
275 Parameters:
276 only_recipe: True to return only the classes inherited by the recipe
277 itself, False to return all classes inherited within
278 the context for the recipe (which includes globally
279 inherited classes).
280 """
281 if only_recipe:
282 global_inherit = [x for x in (self._d.getVar('BBINCLUDED') or '').split() if x.endswith('.bbclass')]
283 else:
284 global_inherit = []
285 for clsfile in self.inherit_files:
286 if only_recipe and clsfile in global_inherit:
287 continue
288 clsname = os.path.splitext(os.path.basename(clsfile))[0]
289 yield clsname
290 def __str__(self):
291 return '%s' % self.pn
292
293
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500294class Tinfoil:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500295 """
296 Tinfoil - an API for scripts and utilities to query
297 BitBake internals and perform build operations.
298 """
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500299
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500300 def __init__(self, output=sys.stdout, tracking=False, setup_logging=True):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500301 """
302 Create a new tinfoil object.
303 Parameters:
304 output: specifies where console output should be sent. Defaults
305 to sys.stdout.
306 tracking: True to enable variable history tracking, False to
307 disable it (default). Enabling this has a minor
308 performance impact so typically it isn't enabled
309 unless you need to query variable history.
310 setup_logging: True to setup a logger so that things like
311 bb.warn() will work immediately and timeout warnings
312 are visible; False to let BitBake do this itself.
313 """
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500314 self.logger = logging.getLogger('BitBake')
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500315 self.config_data = None
316 self.cooker = None
317 self.tracking = tracking
318 self.ui_module = None
319 self.server_connection = None
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500320 self.recipes_parsed = False
321 self.quiet = 0
322 self.oldhandlers = self.logger.handlers[:]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500323 if setup_logging:
324 # This is the *client-side* logger, nothing to do with
325 # logging messages from the server
326 bb.msg.logger_create('BitBake', output)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500327 self.localhandlers = []
328 for handler in self.logger.handlers:
329 if handler not in self.oldhandlers:
330 self.localhandlers.append(handler)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500331
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600332 def __enter__(self):
333 return self
334
335 def __exit__(self, type, value, traceback):
336 self.shutdown()
337
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500338 def prepare(self, config_only=False, config_params=None, quiet=0, extra_features=None):
339 """
340 Prepares the underlying BitBake system to be used via tinfoil.
341 This function must be called prior to calling any of the other
342 functions in the API.
343 NOTE: if you call prepare() you must absolutely call shutdown()
344 before your code terminates. You can use a "with" block to ensure
345 this happens e.g.
346
347 with bb.tinfoil.Tinfoil() as tinfoil:
348 tinfoil.prepare()
349 ...
350
351 Parameters:
352 config_only: True to read only the configuration and not load
353 the cache / parse recipes. This is useful if you just
354 want to query the value of a variable at the global
355 level or you want to do anything else that doesn't
356 involve knowing anything about the recipes in the
357 current configuration. False loads the cache / parses
358 recipes.
359 config_params: optionally specify your own configuration
360 parameters. If not specified an instance of
361 TinfoilConfigParameters will be created internally.
362 quiet: quiet level controlling console output - equivalent
363 to bitbake's -q/--quiet option. Default of 0 gives
364 the same output level as normal bitbake execution.
365 extra_features: extra features to be added to the feature
366 set requested from the server. See
367 CookerFeatures._feature_list for possible
368 features.
369 """
370 self.quiet = quiet
371
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500372 if self.tracking:
373 extrafeatures = [bb.cooker.CookerFeatures.BASEDATASTORE_TRACKING]
374 else:
375 extrafeatures = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500376
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500377 if extra_features:
378 extrafeatures += extra_features
379
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500380 if not config_params:
381 config_params = TinfoilConfigParameters(config_only=config_only, quiet=quiet)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500382
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500383 if not config_only:
384 # Disable local loggers because the UI module is going to set up its own
385 for handler in self.localhandlers:
386 self.logger.handlers.remove(handler)
387 self.localhandlers = []
388
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500389 self.server_connection, ui_module = setup_bitbake(config_params, extrafeatures)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500390
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500391 self.ui_module = ui_module
392
393 # Ensure the path to bitbake's bin directory is in PATH so that things like
394 # bitbake-worker can be run (usually this is the case, but it doesn't have to be)
395 path = os.getenv('PATH').split(':')
396 bitbakebinpath = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', 'bin'))
397 for entry in path:
398 if entry.endswith(os.sep):
399 entry = entry[:-1]
400 if os.path.abspath(entry) == bitbakebinpath:
401 break
402 else:
403 path.insert(0, bitbakebinpath)
404 os.environ['PATH'] = ':'.join(path)
405
406 if self.server_connection:
407 _server_connections.append(self.server_connection)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500408 if config_only:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500409 config_params.updateToServer(self.server_connection.connection, os.environ.copy())
410 self.run_command('parseConfiguration')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500411 else:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500412 self.run_actions(config_params)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500413 self.recipes_parsed = True
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500414
Andrew Geissler82c905d2020-04-13 13:39:40 -0500415 self.config_data = TinfoilDataStoreConnector(self, 0)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500416 self.cooker = TinfoilCookerAdapter(self)
417 self.cooker_data = self.cooker.recipecaches['']
418 else:
419 raise Exception('Failed to start bitbake server')
420
421 def run_actions(self, config_params):
422 """
423 Run the actions specified in config_params through the UI.
424 """
425 ret = self.ui_module.main(self.server_connection.connection, self.server_connection.events, config_params)
426 if ret:
427 raise TinfoilUIException(ret)
428
429 def parseRecipes(self):
430 """
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500431 Legacy function - use parse_recipes() instead.
432 """
433 self.parse_recipes()
434
435 def parse_recipes(self):
436 """
437 Load information on all recipes. Normally you should specify
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500438 config_only=False when calling prepare() instead of using this
439 function; this function is designed for situations where you need
440 to initialise Tinfoil and use it with config_only=True first and
441 then conditionally call this function to parse recipes later.
442 """
443 config_params = TinfoilConfigParameters(config_only=False)
444 self.run_actions(config_params)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500445 self.recipes_parsed = True
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500446
447 def run_command(self, command, *params):
448 """
449 Run a command on the server (as implemented in bb.command).
450 Note that there are two types of command - synchronous and
451 asynchronous; in order to receive the results of asynchronous
452 commands you will need to set an appropriate event mask
453 using set_event_mask() and listen for the result using
454 wait_event() - with the correct event mask you'll at least get
455 bb.command.CommandCompleted and possibly other events before
456 that depending on the command.
457 """
458 if not self.server_connection:
459 raise Exception('Not connected to server (did you call .prepare()?)')
460
461 commandline = [command]
462 if params:
463 commandline.extend(params)
Andrew Geissler4873add2020-11-02 18:44:49 -0600464 result = self.server_connection.connection.runCommand(commandline)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500465 if result[1]:
466 raise TinfoilCommandFailed(result[1])
467 return result[0]
468
469 def set_event_mask(self, eventlist):
470 """Set the event mask which will be applied within wait_event()"""
471 if not self.server_connection:
472 raise Exception('Not connected to server (did you call .prepare()?)')
473 llevel, debug_domains = bb.msg.constructLogOptions()
474 ret = self.run_command('setEventMask', self.server_connection.connection.getEventHandle(), llevel, debug_domains, eventlist)
475 if not ret:
476 raise Exception('setEventMask failed')
477
478 def wait_event(self, timeout=0):
479 """
480 Wait for an event from the server for the specified time.
481 A timeout of 0 means don't wait if there are no events in the queue.
482 Returns the next event in the queue or None if the timeout was
483 reached. Note that in order to recieve any events you will
484 first need to set the internal event mask using set_event_mask()
485 (otherwise whatever event mask the UI set up will be in effect).
486 """
487 if not self.server_connection:
488 raise Exception('Not connected to server (did you call .prepare()?)')
489 return self.server_connection.events.waitEvent(timeout)
490
Andrew Geissler5a43b432020-06-13 10:46:56 -0500491 def get_overlayed_recipes(self, mc=''):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500492 """
493 Find recipes which are overlayed (i.e. where recipes exist in multiple layers)
494 """
Andrew Geissler5a43b432020-06-13 10:46:56 -0500495 return defaultdict(list, self.run_command('getOverlayedRecipes', mc))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500496
497 def get_skipped_recipes(self):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500498 """
499 Find recipes which were skipped (i.e. SkipRecipe was raised
500 during parsing).
501 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500502 return OrderedDict(self.run_command('getSkippedRecipes'))
503
Andrew Geissler82c905d2020-04-13 13:39:40 -0500504 def get_all_providers(self, mc=''):
505 return defaultdict(list, self.run_command('allProviders', mc))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500506
Andrew Geissler82c905d2020-04-13 13:39:40 -0500507 def find_providers(self, mc=''):
508 return self.run_command('findProviders', mc)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500509
510 def find_best_provider(self, pn):
511 return self.run_command('findBestProvider', pn)
512
513 def get_runtime_providers(self, rdep):
514 return self.run_command('getRuntimeProviders', rdep)
515
516 def get_recipe_file(self, pn):
517 """
518 Get the file name for the specified recipe/target. Raises
519 bb.providers.NoProvider if there is no match or the recipe was
520 skipped.
521 """
522 best = self.find_best_provider(pn)
523 if not best or (len(best) > 3 and not best[3]):
524 skiplist = self.get_skipped_recipes()
525 taskdata = bb.taskdata.TaskData(None, skiplist=skiplist)
526 skipreasons = taskdata.get_reasons(pn)
527 if skipreasons:
528 raise bb.providers.NoProvider('%s is unavailable:\n %s' % (pn, ' \n'.join(skipreasons)))
529 else:
530 raise bb.providers.NoProvider('Unable to find any recipe file matching "%s"' % pn)
531 return best[3]
532
Andrew Geissler5a43b432020-06-13 10:46:56 -0500533 def get_file_appends(self, fn, mc=''):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500534 """
535 Find the bbappends for a recipe file
536 """
Andrew Geissler5a43b432020-06-13 10:46:56 -0500537 return self.run_command('getFileAppends', fn, mc)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500538
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500539 def all_recipes(self, mc='', sort=True):
540 """
541 Enable iterating over all recipes in the current configuration.
542 Returns an iterator over TinfoilRecipeInfo objects created on demand.
543 Parameters:
544 mc: The multiconfig, default of '' uses the main configuration.
545 sort: True to sort recipes alphabetically (default), False otherwise
546 """
547 recipecache = self.cooker.recipecaches[mc]
548 if sort:
549 recipes = sorted(recipecache.pkg_pn.items())
550 else:
551 recipes = recipecache.pkg_pn.items()
552 for pn, fns in recipes:
553 prov = self.find_best_provider(pn)
554 recipe = TinfoilRecipeInfo(recipecache,
555 self.config_data,
556 pn=pn,
557 fn=prov[3],
558 fns=fns)
559 yield recipe
560
561 def all_recipe_files(self, mc='', variants=True, preferred_only=False):
562 """
563 Enable iterating over all recipe files in the current configuration.
564 Returns an iterator over file paths.
565 Parameters:
566 mc: The multiconfig, default of '' uses the main configuration.
567 variants: True to include variants of recipes created through
568 BBCLASSEXTEND (default) or False to exclude them
569 preferred_only: True to include only the preferred recipe where
570 multiple exist providing the same PN, False to list
571 all recipes
572 """
573 recipecache = self.cooker.recipecaches[mc]
574 if preferred_only:
575 files = []
576 for pn in recipecache.pkg_pn.keys():
577 prov = self.find_best_provider(pn)
578 files.append(prov[3])
579 else:
580 files = recipecache.pkg_fn.keys()
581 for fn in sorted(files):
582 if not variants and fn.startswith('virtual:'):
583 continue
584 yield fn
585
586
587 def get_recipe_info(self, pn, mc=''):
588 """
589 Get information on a specific recipe in the current configuration by name (PN).
590 Returns a TinfoilRecipeInfo object created on demand.
591 Parameters:
592 mc: The multiconfig, default of '' uses the main configuration.
593 """
594 recipecache = self.cooker.recipecaches[mc]
595 prov = self.find_best_provider(pn)
596 fn = prov[3]
Brad Bishop316dfdd2018-06-25 12:45:53 -0400597 if fn:
598 actual_pn = recipecache.pkg_fn[fn]
599 recipe = TinfoilRecipeInfo(recipecache,
600 self.config_data,
601 pn=actual_pn,
602 fn=fn,
603 fns=recipecache.pkg_pn[actual_pn])
604 return recipe
605 else:
606 return None
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500607
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500608 def parse_recipe(self, pn):
609 """
610 Parse the specified recipe and return a datastore object
611 representing the environment for the recipe.
612 """
613 fn = self.get_recipe_file(pn)
614 return self.parse_recipe_file(fn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500615
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600616 def parse_recipe_file(self, fn, appends=True, appendlist=None, config_data=None):
617 """
618 Parse the specified recipe file (with or without bbappends)
619 and return a datastore object representing the environment
620 for the recipe.
621 Parameters:
622 fn: recipe file to parse - can be a file path or virtual
623 specification
624 appends: True to apply bbappends, False otherwise
625 appendlist: optional list of bbappend files to apply, if you
626 want to filter them
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600627 """
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500628 if self.tracking:
629 # Enable history tracking just for the parse operation
630 self.run_command('enableDataTracking')
631 try:
632 if appends and appendlist == []:
633 appends = False
634 if config_data:
Andrew Geissler82c905d2020-04-13 13:39:40 -0500635 config_data = bb.data.createCopy(config_data)
636 dscon = self.run_command('parseRecipeFile', fn, appends, appendlist, config_data.dsindex)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500637 else:
638 dscon = self.run_command('parseRecipeFile', fn, appends, appendlist)
639 if dscon:
640 return self._reconvert_type(dscon, 'DataStoreConnectionHandle')
641 else:
642 return None
643 finally:
644 if self.tracking:
645 self.run_command('disableDataTracking')
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500646
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500647 def build_file(self, buildfile, task, internal=True):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500648 """
649 Runs the specified task for just a single recipe (i.e. no dependencies).
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500650 This is equivalent to bitbake -b, except with the default internal=True
651 no warning about dependencies will be produced, normal info messages
652 from the runqueue will be silenced and BuildInit, BuildStarted and
653 BuildCompleted events will not be fired.
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500654 """
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500655 return self.run_command('buildFile', buildfile, task, internal)
656
657 def build_targets(self, targets, task=None, handle_events=True, extra_events=None, event_callback=None):
658 """
659 Builds the specified targets. This is equivalent to a normal invocation
660 of bitbake. Has built-in event handling which is enabled by default and
661 can be extended if needed.
662 Parameters:
663 targets:
664 One or more targets to build. Can be a list or a
665 space-separated string.
666 task:
667 The task to run; if None then the value of BB_DEFAULT_TASK
668 will be used. Default None.
669 handle_events:
670 True to handle events in a similar way to normal bitbake
671 invocation with knotty; False to return immediately (on the
672 assumption that the caller will handle the events instead).
673 Default True.
674 extra_events:
675 An optional list of events to add to the event mask (if
676 handle_events=True). If you add events here you also need
677 to specify a callback function in event_callback that will
678 handle the additional events. Default None.
679 event_callback:
680 An optional function taking a single parameter which
681 will be called first upon receiving any event (if
682 handle_events=True) so that the caller can override or
683 extend the event handling. Default None.
684 """
685 if isinstance(targets, str):
686 targets = targets.split()
687 if not task:
688 task = self.config_data.getVar('BB_DEFAULT_TASK')
689
690 if handle_events:
691 # A reasonable set of default events matching up with those we handle below
692 eventmask = [
693 'bb.event.BuildStarted',
694 'bb.event.BuildCompleted',
695 'logging.LogRecord',
696 'bb.event.NoProvider',
697 'bb.command.CommandCompleted',
698 'bb.command.CommandFailed',
699 'bb.build.TaskStarted',
700 'bb.build.TaskFailed',
701 'bb.build.TaskSucceeded',
702 'bb.build.TaskFailedSilent',
703 'bb.build.TaskProgress',
704 'bb.runqueue.runQueueTaskStarted',
705 'bb.runqueue.sceneQueueTaskStarted',
706 'bb.event.ProcessStarted',
707 'bb.event.ProcessProgress',
708 'bb.event.ProcessFinished',
709 ]
710 if extra_events:
711 eventmask.extend(extra_events)
712 ret = self.set_event_mask(eventmask)
713
714 includelogs = self.config_data.getVar('BBINCLUDELOGS')
715 loglines = self.config_data.getVar('BBINCLUDELOGS_LINES')
716
717 ret = self.run_command('buildTargets', targets, task)
718 if handle_events:
719 result = False
720 # Borrowed from knotty, instead somewhat hackily we use the helper
721 # as the object to store "shutdown" on
722 helper = bb.ui.uihelper.BBUIHelper()
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500723 helper.shutdown = 0
724 parseprogress = None
Andrew Geissler82c905d2020-04-13 13:39:40 -0500725 termfilter = bb.ui.knotty.TerminalFilter(helper, helper, self.logger.handlers, quiet=self.quiet)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500726 try:
727 while True:
728 try:
729 event = self.wait_event(0.25)
730 if event:
731 if event_callback and event_callback(event):
732 continue
733 if helper.eventHandler(event):
734 if isinstance(event, bb.build.TaskFailedSilent):
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500735 self.logger.warning("Logfile for failed setscene task is %s" % event.logfile)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500736 elif isinstance(event, bb.build.TaskFailed):
737 bb.ui.knotty.print_event_log(event, includelogs, loglines, termfilter)
738 continue
739 if isinstance(event, bb.event.ProcessStarted):
740 if self.quiet > 1:
741 continue
742 parseprogress = bb.ui.knotty.new_progress(event.processname, event.total)
743 parseprogress.start(False)
744 continue
745 if isinstance(event, bb.event.ProcessProgress):
746 if self.quiet > 1:
747 continue
748 if parseprogress:
749 parseprogress.update(event.progress)
750 else:
751 bb.warn("Got ProcessProgress event for someting that never started?")
752 continue
753 if isinstance(event, bb.event.ProcessFinished):
754 if self.quiet > 1:
755 continue
756 if parseprogress:
757 parseprogress.finish()
758 parseprogress = None
759 continue
760 if isinstance(event, bb.command.CommandCompleted):
761 result = True
762 break
763 if isinstance(event, bb.command.CommandFailed):
764 self.logger.error(str(event))
765 result = False
766 break
767 if isinstance(event, logging.LogRecord):
768 if event.taskpid == 0 or event.levelno > logging.INFO:
769 self.logger.handle(event)
770 continue
771 if isinstance(event, bb.event.NoProvider):
772 self.logger.error(str(event))
773 result = False
774 break
775
776 elif helper.shutdown > 1:
777 break
778 termfilter.updateFooter()
779 except KeyboardInterrupt:
780 termfilter.clearFooter()
781 if helper.shutdown == 1:
782 print("\nSecond Keyboard Interrupt, stopping...\n")
783 ret = self.run_command("stateForceShutdown")
784 if ret and ret[2]:
785 self.logger.error("Unable to cleanly stop: %s" % ret[2])
786 elif helper.shutdown == 0:
787 print("\nKeyboard Interrupt, closing down...\n")
788 interrupted = True
789 ret = self.run_command("stateShutdown")
790 if ret and ret[2]:
791 self.logger.error("Unable to cleanly shutdown: %s" % ret[2])
792 helper.shutdown = helper.shutdown + 1
793 termfilter.clearFooter()
794 finally:
795 termfilter.finish()
796 if helper.failed_tasks:
797 result = False
798 return result
799 else:
800 return ret
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600801
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500802 def shutdown(self):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500803 """
804 Shut down tinfoil. Disconnects from the server and gracefully
805 releases any associated resources. You must call this function if
806 prepare() has been called, or use a with... block when you create
807 the tinfoil object which will ensure that it gets called.
808 """
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500809 try:
810 if self.server_connection:
811 try:
812 self.run_command('clientComplete')
813 finally:
814 _server_connections.remove(self.server_connection)
815 bb.event.ui_queue = []
816 self.server_connection.terminate()
817 self.server_connection = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500818
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500819 finally:
820 # Restore logging handlers to how it looked when we started
821 if self.oldhandlers:
822 for handler in self.logger.handlers:
823 if handler not in self.oldhandlers:
824 self.logger.handlers.remove(handler)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500825
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500826 def _reconvert_type(self, obj, origtypename):
827 """
828 Convert an object back to the right type, in the case
829 that marshalling has changed it (especially with xmlrpc)
830 """
831 supported_types = {
832 'set': set,
833 'DataStoreConnectionHandle': bb.command.DataStoreConnectionHandle,
834 }
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500835
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500836 origtype = supported_types.get(origtypename, None)
837 if origtype is None:
838 raise Exception('Unsupported type "%s"' % origtypename)
839 if type(obj) == origtype:
840 newobj = obj
841 elif isinstance(obj, dict):
842 # New style class
843 newobj = origtype()
844 for k,v in obj.items():
845 setattr(newobj, k, v)
846 else:
847 # Assume we can coerce the type
848 newobj = origtype(obj)
849
850 if isinstance(newobj, bb.command.DataStoreConnectionHandle):
Andrew Geissler82c905d2020-04-13 13:39:40 -0500851 newobj = TinfoilDataStoreConnector(self, newobj.dsindex)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500852
853 return newobj
854
855
856class TinfoilConfigParameters(BitBakeConfigParameters):
857
858 def __init__(self, config_only, **options):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500859 self.initial_options = options
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500860 # Apply some sane defaults
861 if not 'parse_only' in options:
862 self.initial_options['parse_only'] = not config_only
863 #if not 'status_only' in options:
864 # self.initial_options['status_only'] = config_only
865 if not 'ui' in options:
866 self.initial_options['ui'] = 'knotty'
867 if not 'argv' in options:
868 self.initial_options['argv'] = []
869
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500870 super(TinfoilConfigParameters, self).__init__()
871
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500872 def parseCommandLine(self, argv=None):
873 # We don't want any parameters parsed from the command line
874 opts = super(TinfoilConfigParameters, self).parseCommandLine([])
875 for key, val in self.initial_options.items():
876 setattr(opts[0], key, val)
877 return opts