blob: e68a3b879a8dd6d7f20e50937672b31b4eb725ec [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# tinfoil: a simple wrapper around cooker for bitbake-based command-line utilities
2#
Brad Bishop6e60e8b2018-02-01 10:27:11 -05003# Copyright (C) 2012-2017 Intel Corporation
Patrick Williamsc124f4f2015-09-15 14:41:29 -05004# Copyright (C) 2011 Mentor Graphics Corporation
Brad Bishopd7bf8c12018-02-25 22:55:05 -05005# Copyright (C) 2006-2012 Richard Purdie
Patrick Williamsc124f4f2015-09-15 14:41:29 -05006#
Brad Bishopc342db32019-05-15 21:57:59 -04007# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -05008#
Patrick Williamsc124f4f2015-09-15 14:41:29 -05009
10import logging
Patrick Williamsc124f4f2015-09-15 14:41:29 -050011import os
12import sys
Brad Bishop6e60e8b2018-02-01 10:27:11 -050013import atexit
14import re
15from collections import OrderedDict, defaultdict
Andrew Geissler82c905d2020-04-13 13:39:40 -050016from functools import partial
Patrick Williamsc124f4f2015-09-15 14:41:29 -050017
18import bb.cache
19import bb.cooker
20import bb.providers
Brad Bishop6e60e8b2018-02-01 10:27:11 -050021import bb.taskdata
Patrick Williamsc124f4f2015-09-15 14:41:29 -050022import bb.utils
Brad Bishop6e60e8b2018-02-01 10:27:11 -050023import bb.command
24import bb.remotedata
Andrew Geissler82c905d2020-04-13 13:39:40 -050025from bb.main import setup_bitbake, BitBakeConfigParameters
Patrick Williamsc124f4f2015-09-15 14:41:29 -050026import bb.fetch2
27
Brad Bishop6e60e8b2018-02-01 10:27:11 -050028
29# We need this in order to shut down the connection to the bitbake server,
30# otherwise the process will never properly exit
31_server_connections = []
32def _terminate_connections():
33 for connection in _server_connections:
34 connection.terminate()
35atexit.register(_terminate_connections)
36
37class TinfoilUIException(Exception):
38 """Exception raised when the UI returns non-zero from its main function"""
39 def __init__(self, returncode):
40 self.returncode = returncode
41 def __repr__(self):
42 return 'UI module main returned %d' % self.returncode
43
44class TinfoilCommandFailed(Exception):
45 """Exception raised when run_command fails"""
46
Andrew Geissler82c905d2020-04-13 13:39:40 -050047class TinfoilDataStoreConnectorVarHistory:
48 def __init__(self, tinfoil, dsindex):
49 self.tinfoil = tinfoil
50 self.dsindex = dsindex
51
52 def remoteCommand(self, cmd, *args, **kwargs):
53 return self.tinfoil.run_command('dataStoreConnectorVarHistCmd', self.dsindex, cmd, args, kwargs)
54
Andrew Geisslerc926e172021-05-07 16:11:35 -050055 def emit(self, var, oval, val, o, d):
56 ret = self.tinfoil.run_command('dataStoreConnectorVarHistCmdEmit', self.dsindex, var, oval, val, d.dsindex)
57 o.write(ret)
58
Andrew Geissler82c905d2020-04-13 13:39:40 -050059 def __getattr__(self, name):
60 if not hasattr(bb.data_smart.VariableHistory, name):
61 raise AttributeError("VariableHistory has no such method %s" % name)
62
63 newfunc = partial(self.remoteCommand, name)
64 setattr(self, name, newfunc)
65 return newfunc
66
67class TinfoilDataStoreConnectorIncHistory:
68 def __init__(self, tinfoil, dsindex):
69 self.tinfoil = tinfoil
70 self.dsindex = dsindex
71
72 def remoteCommand(self, cmd, *args, **kwargs):
73 return self.tinfoil.run_command('dataStoreConnectorIncHistCmd', self.dsindex, cmd, args, kwargs)
74
75 def __getattr__(self, name):
76 if not hasattr(bb.data_smart.IncludeHistory, name):
77 raise AttributeError("IncludeHistory has no such method %s" % name)
78
79 newfunc = partial(self.remoteCommand, name)
80 setattr(self, name, newfunc)
81 return newfunc
82
Brad Bishop6e60e8b2018-02-01 10:27:11 -050083class TinfoilDataStoreConnector:
Andrew Geissler82c905d2020-04-13 13:39:40 -050084 """
85 Connector object used to enable access to datastore objects via tinfoil
86 Method calls are transmitted to the remote datastore for processing, if a datastore is
87 returned we return a connector object for the new store
88 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -050089
90 def __init__(self, tinfoil, dsindex):
91 self.tinfoil = tinfoil
92 self.dsindex = dsindex
Andrew Geissler82c905d2020-04-13 13:39:40 -050093 self.varhistory = TinfoilDataStoreConnectorVarHistory(tinfoil, dsindex)
94 self.inchistory = TinfoilDataStoreConnectorIncHistory(tinfoil, dsindex)
95
96 def remoteCommand(self, cmd, *args, **kwargs):
97 ret = self.tinfoil.run_command('dataStoreConnectorCmd', self.dsindex, cmd, args, kwargs)
98 if isinstance(ret, bb.command.DataStoreConnectionHandle):
99 return TinfoilDataStoreConnector(self.tinfoil, ret.dsindex)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500100 return ret
Andrew Geissler82c905d2020-04-13 13:39:40 -0500101
102 def __getattr__(self, name):
103 if not hasattr(bb.data._dict_type, name):
104 raise AttributeError("Data store has no such method %s" % name)
105
106 newfunc = partial(self.remoteCommand, name)
107 setattr(self, name, newfunc)
108 return newfunc
109
110 def __iter__(self):
111 keys = self.tinfoil.run_command('dataStoreConnectorCmd', self.dsindex, "keys", [], {})
112 for k in keys:
113 yield k
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500114
115class TinfoilCookerAdapter:
116 """
117 Provide an adapter for existing code that expects to access a cooker object via Tinfoil,
118 since now Tinfoil is on the client side it no longer has direct access.
119 """
120
121 class TinfoilCookerCollectionAdapter:
122 """ cooker.collection adapter """
Andrew Geissler5a43b432020-06-13 10:46:56 -0500123 def __init__(self, tinfoil, mc=''):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500124 self.tinfoil = tinfoil
Andrew Geissler5a43b432020-06-13 10:46:56 -0500125 self.mc = mc
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500126 def get_file_appends(self, fn):
Andrew Geissler5a43b432020-06-13 10:46:56 -0500127 return self.tinfoil.get_file_appends(fn, self.mc)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500128 def __getattr__(self, name):
129 if name == 'overlayed':
Andrew Geissler5a43b432020-06-13 10:46:56 -0500130 return self.tinfoil.get_overlayed_recipes(self.mc)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500131 elif name == 'bbappends':
Andrew Geissler5a43b432020-06-13 10:46:56 -0500132 return self.tinfoil.run_command('getAllAppends', self.mc)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500133 else:
134 raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
135
136 class TinfoilRecipeCacheAdapter:
137 """ cooker.recipecache adapter """
Andrew Geissler82c905d2020-04-13 13:39:40 -0500138 def __init__(self, tinfoil, mc=''):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500139 self.tinfoil = tinfoil
Andrew Geissler82c905d2020-04-13 13:39:40 -0500140 self.mc = mc
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500141 self._cache = {}
142
143 def get_pkg_pn_fn(self):
Andrew Geissler82c905d2020-04-13 13:39:40 -0500144 pkg_pn = defaultdict(list, self.tinfoil.run_command('getRecipes', self.mc) or [])
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500145 pkg_fn = {}
146 for pn, fnlist in pkg_pn.items():
147 for fn in fnlist:
148 pkg_fn[fn] = pn
149 self._cache['pkg_pn'] = pkg_pn
150 self._cache['pkg_fn'] = pkg_fn
151
152 def __getattr__(self, name):
153 # Grab these only when they are requested since they aren't always used
154 if name in self._cache:
155 return self._cache[name]
156 elif name == 'pkg_pn':
157 self.get_pkg_pn_fn()
158 return self._cache[name]
159 elif name == 'pkg_fn':
160 self.get_pkg_pn_fn()
161 return self._cache[name]
162 elif name == 'deps':
Andrew Geissler82c905d2020-04-13 13:39:40 -0500163 attrvalue = defaultdict(list, self.tinfoil.run_command('getRecipeDepends', self.mc) or [])
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500164 elif name == 'rundeps':
Andrew Geissler82c905d2020-04-13 13:39:40 -0500165 attrvalue = defaultdict(lambda: defaultdict(list), self.tinfoil.run_command('getRuntimeDepends', self.mc) or [])
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500166 elif name == 'runrecs':
Andrew Geissler82c905d2020-04-13 13:39:40 -0500167 attrvalue = defaultdict(lambda: defaultdict(list), self.tinfoil.run_command('getRuntimeRecommends', self.mc) or [])
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500168 elif name == 'pkg_pepvpr':
Andrew Geissler82c905d2020-04-13 13:39:40 -0500169 attrvalue = self.tinfoil.run_command('getRecipeVersions', self.mc) or {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500170 elif name == 'inherits':
Andrew Geissler82c905d2020-04-13 13:39:40 -0500171 attrvalue = self.tinfoil.run_command('getRecipeInherits', self.mc) or {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500172 elif name == 'bbfile_priority':
Andrew Geissler82c905d2020-04-13 13:39:40 -0500173 attrvalue = self.tinfoil.run_command('getBbFilePriority', self.mc) or {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500174 elif name == 'pkg_dp':
Andrew Geissler82c905d2020-04-13 13:39:40 -0500175 attrvalue = self.tinfoil.run_command('getDefaultPreference', self.mc) or {}
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500176 elif name == 'fn_provides':
Andrew Geissler82c905d2020-04-13 13:39:40 -0500177 attrvalue = self.tinfoil.run_command('getRecipeProvides', self.mc) or {}
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500178 elif name == 'packages':
Andrew Geissler82c905d2020-04-13 13:39:40 -0500179 attrvalue = self.tinfoil.run_command('getRecipePackages', self.mc) or {}
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500180 elif name == 'packages_dynamic':
Andrew Geissler82c905d2020-04-13 13:39:40 -0500181 attrvalue = self.tinfoil.run_command('getRecipePackagesDynamic', self.mc) or {}
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500182 elif name == 'rproviders':
Andrew Geissler82c905d2020-04-13 13:39:40 -0500183 attrvalue = self.tinfoil.run_command('getRProviders', self.mc) or {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500184 else:
185 raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
186
187 self._cache[name] = attrvalue
188 return attrvalue
189
190 def __init__(self, tinfoil):
191 self.tinfoil = tinfoil
Andrew Geissler5a43b432020-06-13 10:46:56 -0500192 self.multiconfigs = [''] + (tinfoil.config_data.getVar('BBMULTICONFIG') or '').split()
193 self.collections = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500194 self.recipecaches = {}
Andrew Geissler5a43b432020-06-13 10:46:56 -0500195 for mc in self.multiconfigs:
196 self.collections[mc] = self.TinfoilCookerCollectionAdapter(tinfoil, mc)
Andrew Geissler82c905d2020-04-13 13:39:40 -0500197 self.recipecaches[mc] = self.TinfoilRecipeCacheAdapter(tinfoil, mc)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500198 self._cache = {}
199 def __getattr__(self, name):
200 # Grab these only when they are requested since they aren't always used
201 if name in self._cache:
202 return self._cache[name]
203 elif name == 'skiplist':
204 attrvalue = self.tinfoil.get_skipped_recipes()
205 elif name == 'bbfile_config_priorities':
206 ret = self.tinfoil.run_command('getLayerPriorities')
207 bbfile_config_priorities = []
208 for collection, pattern, regex, pri in ret:
209 bbfile_config_priorities.append((collection, pattern, re.compile(regex), pri))
210
211 attrvalue = bbfile_config_priorities
212 else:
213 raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
214
215 self._cache[name] = attrvalue
216 return attrvalue
217
218 def findBestProvider(self, pn):
219 return self.tinfoil.find_best_provider(pn)
220
221
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500222class TinfoilRecipeInfo:
223 """
224 Provides a convenient representation of the cached information for a single recipe.
225 Some attributes are set on construction, others are read on-demand (which internally
226 may result in a remote procedure call to the bitbake server the first time).
227 Note that only information which is cached is available through this object - if
228 you need other variable values you will need to parse the recipe using
229 Tinfoil.parse_recipe().
230 """
231 def __init__(self, recipecache, d, pn, fn, fns):
232 self._recipecache = recipecache
233 self._d = d
234 self.pn = pn
235 self.fn = fn
236 self.fns = fns
237 self.inherit_files = recipecache.inherits[fn]
238 self.depends = recipecache.deps[fn]
239 (self.pe, self.pv, self.pr) = recipecache.pkg_pepvpr[fn]
240 self._cached_packages = None
241 self._cached_rprovides = None
242 self._cached_packages_dynamic = None
243
244 def __getattr__(self, name):
245 if name == 'alternates':
246 return [x for x in self.fns if x != self.fn]
247 elif name == 'rdepends':
248 return self._recipecache.rundeps[self.fn]
249 elif name == 'rrecommends':
250 return self._recipecache.runrecs[self.fn]
251 elif name == 'provides':
252 return self._recipecache.fn_provides[self.fn]
253 elif name == 'packages':
254 if self._cached_packages is None:
255 self._cached_packages = []
256 for pkg, fns in self._recipecache.packages.items():
257 if self.fn in fns:
258 self._cached_packages.append(pkg)
259 return self._cached_packages
260 elif name == 'packages_dynamic':
261 if self._cached_packages_dynamic is None:
262 self._cached_packages_dynamic = []
263 for pkg, fns in self._recipecache.packages_dynamic.items():
264 if self.fn in fns:
265 self._cached_packages_dynamic.append(pkg)
266 return self._cached_packages_dynamic
267 elif name == 'rprovides':
268 if self._cached_rprovides is None:
269 self._cached_rprovides = []
270 for pkg, fns in self._recipecache.rproviders.items():
271 if self.fn in fns:
272 self._cached_rprovides.append(pkg)
273 return self._cached_rprovides
274 else:
275 raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
276 def inherits(self, only_recipe=False):
277 """
278 Get the inherited classes for a recipe. Returns the class names only.
279 Parameters:
280 only_recipe: True to return only the classes inherited by the recipe
281 itself, False to return all classes inherited within
282 the context for the recipe (which includes globally
283 inherited classes).
284 """
285 if only_recipe:
286 global_inherit = [x for x in (self._d.getVar('BBINCLUDED') or '').split() if x.endswith('.bbclass')]
287 else:
288 global_inherit = []
289 for clsfile in self.inherit_files:
290 if only_recipe and clsfile in global_inherit:
291 continue
292 clsname = os.path.splitext(os.path.basename(clsfile))[0]
293 yield clsname
294 def __str__(self):
295 return '%s' % self.pn
296
297
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500298class Tinfoil:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500299 """
300 Tinfoil - an API for scripts and utilities to query
301 BitBake internals and perform build operations.
302 """
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500303
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500304 def __init__(self, output=sys.stdout, tracking=False, setup_logging=True):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500305 """
306 Create a new tinfoil object.
307 Parameters:
308 output: specifies where console output should be sent. Defaults
309 to sys.stdout.
310 tracking: True to enable variable history tracking, False to
311 disable it (default). Enabling this has a minor
312 performance impact so typically it isn't enabled
313 unless you need to query variable history.
314 setup_logging: True to setup a logger so that things like
315 bb.warn() will work immediately and timeout warnings
316 are visible; False to let BitBake do this itself.
317 """
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500318 self.logger = logging.getLogger('BitBake')
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500319 self.config_data = None
320 self.cooker = None
321 self.tracking = tracking
322 self.ui_module = None
323 self.server_connection = None
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500324 self.recipes_parsed = False
325 self.quiet = 0
326 self.oldhandlers = self.logger.handlers[:]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500327 if setup_logging:
328 # This is the *client-side* logger, nothing to do with
329 # logging messages from the server
330 bb.msg.logger_create('BitBake', output)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500331 self.localhandlers = []
332 for handler in self.logger.handlers:
333 if handler not in self.oldhandlers:
334 self.localhandlers.append(handler)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500335
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600336 def __enter__(self):
337 return self
338
339 def __exit__(self, type, value, traceback):
340 self.shutdown()
341
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500342 def prepare(self, config_only=False, config_params=None, quiet=0, extra_features=None):
343 """
344 Prepares the underlying BitBake system to be used via tinfoil.
345 This function must be called prior to calling any of the other
346 functions in the API.
347 NOTE: if you call prepare() you must absolutely call shutdown()
348 before your code terminates. You can use a "with" block to ensure
349 this happens e.g.
350
351 with bb.tinfoil.Tinfoil() as tinfoil:
352 tinfoil.prepare()
353 ...
354
355 Parameters:
356 config_only: True to read only the configuration and not load
357 the cache / parse recipes. This is useful if you just
358 want to query the value of a variable at the global
359 level or you want to do anything else that doesn't
360 involve knowing anything about the recipes in the
361 current configuration. False loads the cache / parses
362 recipes.
363 config_params: optionally specify your own configuration
364 parameters. If not specified an instance of
365 TinfoilConfigParameters will be created internally.
366 quiet: quiet level controlling console output - equivalent
367 to bitbake's -q/--quiet option. Default of 0 gives
368 the same output level as normal bitbake execution.
369 extra_features: extra features to be added to the feature
370 set requested from the server. See
371 CookerFeatures._feature_list for possible
372 features.
373 """
374 self.quiet = quiet
375
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500376 if self.tracking:
377 extrafeatures = [bb.cooker.CookerFeatures.BASEDATASTORE_TRACKING]
378 else:
379 extrafeatures = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500380
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500381 if extra_features:
382 extrafeatures += extra_features
383
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500384 if not config_params:
385 config_params = TinfoilConfigParameters(config_only=config_only, quiet=quiet)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500386
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500387 if not config_only:
388 # Disable local loggers because the UI module is going to set up its own
389 for handler in self.localhandlers:
390 self.logger.handlers.remove(handler)
391 self.localhandlers = []
392
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500393 self.server_connection, ui_module = setup_bitbake(config_params, extrafeatures)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500394
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500395 self.ui_module = ui_module
396
397 # Ensure the path to bitbake's bin directory is in PATH so that things like
398 # bitbake-worker can be run (usually this is the case, but it doesn't have to be)
399 path = os.getenv('PATH').split(':')
400 bitbakebinpath = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', 'bin'))
401 for entry in path:
402 if entry.endswith(os.sep):
403 entry = entry[:-1]
404 if os.path.abspath(entry) == bitbakebinpath:
405 break
406 else:
407 path.insert(0, bitbakebinpath)
408 os.environ['PATH'] = ':'.join(path)
409
410 if self.server_connection:
411 _server_connections.append(self.server_connection)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500412 if config_only:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500413 config_params.updateToServer(self.server_connection.connection, os.environ.copy())
414 self.run_command('parseConfiguration')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500415 else:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500416 self.run_actions(config_params)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500417 self.recipes_parsed = True
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500418
Andrew Geissler82c905d2020-04-13 13:39:40 -0500419 self.config_data = TinfoilDataStoreConnector(self, 0)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500420 self.cooker = TinfoilCookerAdapter(self)
421 self.cooker_data = self.cooker.recipecaches['']
422 else:
423 raise Exception('Failed to start bitbake server')
424
425 def run_actions(self, config_params):
426 """
427 Run the actions specified in config_params through the UI.
428 """
429 ret = self.ui_module.main(self.server_connection.connection, self.server_connection.events, config_params)
430 if ret:
431 raise TinfoilUIException(ret)
432
433 def parseRecipes(self):
434 """
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500435 Legacy function - use parse_recipes() instead.
436 """
437 self.parse_recipes()
438
439 def parse_recipes(self):
440 """
441 Load information on all recipes. Normally you should specify
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500442 config_only=False when calling prepare() instead of using this
443 function; this function is designed for situations where you need
444 to initialise Tinfoil and use it with config_only=True first and
445 then conditionally call this function to parse recipes later.
446 """
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500447 config_params = TinfoilConfigParameters(config_only=False, quiet=self.quiet)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500448 self.run_actions(config_params)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500449 self.recipes_parsed = True
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500450
Andrew Geissler9aee5002022-03-30 16:27:02 +0000451 def run_command(self, command, *params, handle_events=True):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500452 """
453 Run a command on the server (as implemented in bb.command).
454 Note that there are two types of command - synchronous and
455 asynchronous; in order to receive the results of asynchronous
456 commands you will need to set an appropriate event mask
457 using set_event_mask() and listen for the result using
458 wait_event() - with the correct event mask you'll at least get
459 bb.command.CommandCompleted and possibly other events before
460 that depending on the command.
461 """
462 if not self.server_connection:
463 raise Exception('Not connected to server (did you call .prepare()?)')
464
465 commandline = [command]
466 if params:
467 commandline.extend(params)
Andrew Geisslerf0343792020-11-18 10:42:21 -0600468 try:
469 result = self.server_connection.connection.runCommand(commandline)
470 finally:
Andrew Geissler9aee5002022-03-30 16:27:02 +0000471 while handle_events:
Andrew Geisslerf0343792020-11-18 10:42:21 -0600472 event = self.wait_event()
473 if not event:
474 break
475 if isinstance(event, logging.LogRecord):
476 if event.taskpid == 0 or event.levelno > logging.INFO:
477 self.logger.handle(event)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500478 if result[1]:
479 raise TinfoilCommandFailed(result[1])
480 return result[0]
481
482 def set_event_mask(self, eventlist):
483 """Set the event mask which will be applied within wait_event()"""
484 if not self.server_connection:
485 raise Exception('Not connected to server (did you call .prepare()?)')
486 llevel, debug_domains = bb.msg.constructLogOptions()
487 ret = self.run_command('setEventMask', self.server_connection.connection.getEventHandle(), llevel, debug_domains, eventlist)
488 if not ret:
489 raise Exception('setEventMask failed')
490
491 def wait_event(self, timeout=0):
492 """
493 Wait for an event from the server for the specified time.
494 A timeout of 0 means don't wait if there are no events in the queue.
495 Returns the next event in the queue or None if the timeout was
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000496 reached. Note that in order to receive any events you will
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500497 first need to set the internal event mask using set_event_mask()
498 (otherwise whatever event mask the UI set up will be in effect).
499 """
500 if not self.server_connection:
501 raise Exception('Not connected to server (did you call .prepare()?)')
502 return self.server_connection.events.waitEvent(timeout)
503
Andrew Geissler5a43b432020-06-13 10:46:56 -0500504 def get_overlayed_recipes(self, mc=''):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500505 """
506 Find recipes which are overlayed (i.e. where recipes exist in multiple layers)
507 """
Andrew Geissler5a43b432020-06-13 10:46:56 -0500508 return defaultdict(list, self.run_command('getOverlayedRecipes', mc))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500509
510 def get_skipped_recipes(self):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500511 """
512 Find recipes which were skipped (i.e. SkipRecipe was raised
513 during parsing).
514 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500515 return OrderedDict(self.run_command('getSkippedRecipes'))
516
Andrew Geissler82c905d2020-04-13 13:39:40 -0500517 def get_all_providers(self, mc=''):
518 return defaultdict(list, self.run_command('allProviders', mc))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500519
Andrew Geissler82c905d2020-04-13 13:39:40 -0500520 def find_providers(self, mc=''):
521 return self.run_command('findProviders', mc)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500522
523 def find_best_provider(self, pn):
524 return self.run_command('findBestProvider', pn)
525
526 def get_runtime_providers(self, rdep):
527 return self.run_command('getRuntimeProviders', rdep)
528
529 def get_recipe_file(self, pn):
530 """
531 Get the file name for the specified recipe/target. Raises
532 bb.providers.NoProvider if there is no match or the recipe was
533 skipped.
534 """
535 best = self.find_best_provider(pn)
536 if not best or (len(best) > 3 and not best[3]):
537 skiplist = self.get_skipped_recipes()
538 taskdata = bb.taskdata.TaskData(None, skiplist=skiplist)
539 skipreasons = taskdata.get_reasons(pn)
540 if skipreasons:
541 raise bb.providers.NoProvider('%s is unavailable:\n %s' % (pn, ' \n'.join(skipreasons)))
542 else:
543 raise bb.providers.NoProvider('Unable to find any recipe file matching "%s"' % pn)
544 return best[3]
545
Andrew Geissler5a43b432020-06-13 10:46:56 -0500546 def get_file_appends(self, fn, mc=''):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500547 """
548 Find the bbappends for a recipe file
549 """
Andrew Geissler5a43b432020-06-13 10:46:56 -0500550 return self.run_command('getFileAppends', fn, mc)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500551
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500552 def all_recipes(self, mc='', sort=True):
553 """
554 Enable iterating over all recipes in the current configuration.
555 Returns an iterator over TinfoilRecipeInfo objects created on demand.
556 Parameters:
557 mc: The multiconfig, default of '' uses the main configuration.
558 sort: True to sort recipes alphabetically (default), False otherwise
559 """
560 recipecache = self.cooker.recipecaches[mc]
561 if sort:
562 recipes = sorted(recipecache.pkg_pn.items())
563 else:
564 recipes = recipecache.pkg_pn.items()
565 for pn, fns in recipes:
566 prov = self.find_best_provider(pn)
567 recipe = TinfoilRecipeInfo(recipecache,
568 self.config_data,
569 pn=pn,
570 fn=prov[3],
571 fns=fns)
572 yield recipe
573
574 def all_recipe_files(self, mc='', variants=True, preferred_only=False):
575 """
576 Enable iterating over all recipe files in the current configuration.
577 Returns an iterator over file paths.
578 Parameters:
579 mc: The multiconfig, default of '' uses the main configuration.
580 variants: True to include variants of recipes created through
581 BBCLASSEXTEND (default) or False to exclude them
582 preferred_only: True to include only the preferred recipe where
583 multiple exist providing the same PN, False to list
584 all recipes
585 """
586 recipecache = self.cooker.recipecaches[mc]
587 if preferred_only:
588 files = []
589 for pn in recipecache.pkg_pn.keys():
590 prov = self.find_best_provider(pn)
591 files.append(prov[3])
592 else:
593 files = recipecache.pkg_fn.keys()
594 for fn in sorted(files):
595 if not variants and fn.startswith('virtual:'):
596 continue
597 yield fn
598
599
600 def get_recipe_info(self, pn, mc=''):
601 """
602 Get information on a specific recipe in the current configuration by name (PN).
603 Returns a TinfoilRecipeInfo object created on demand.
604 Parameters:
605 mc: The multiconfig, default of '' uses the main configuration.
606 """
607 recipecache = self.cooker.recipecaches[mc]
608 prov = self.find_best_provider(pn)
609 fn = prov[3]
Brad Bishop316dfdd2018-06-25 12:45:53 -0400610 if fn:
611 actual_pn = recipecache.pkg_fn[fn]
612 recipe = TinfoilRecipeInfo(recipecache,
613 self.config_data,
614 pn=actual_pn,
615 fn=fn,
616 fns=recipecache.pkg_pn[actual_pn])
617 return recipe
618 else:
619 return None
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500620
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500621 def parse_recipe(self, pn):
622 """
623 Parse the specified recipe and return a datastore object
624 representing the environment for the recipe.
625 """
626 fn = self.get_recipe_file(pn)
627 return self.parse_recipe_file(fn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500628
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600629 def parse_recipe_file(self, fn, appends=True, appendlist=None, config_data=None):
630 """
631 Parse the specified recipe file (with or without bbappends)
632 and return a datastore object representing the environment
633 for the recipe.
634 Parameters:
635 fn: recipe file to parse - can be a file path or virtual
636 specification
637 appends: True to apply bbappends, False otherwise
638 appendlist: optional list of bbappend files to apply, if you
639 want to filter them
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600640 """
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500641 if self.tracking:
642 # Enable history tracking just for the parse operation
643 self.run_command('enableDataTracking')
644 try:
645 if appends and appendlist == []:
646 appends = False
647 if config_data:
Andrew Geissler82c905d2020-04-13 13:39:40 -0500648 config_data = bb.data.createCopy(config_data)
649 dscon = self.run_command('parseRecipeFile', fn, appends, appendlist, config_data.dsindex)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500650 else:
651 dscon = self.run_command('parseRecipeFile', fn, appends, appendlist)
652 if dscon:
653 return self._reconvert_type(dscon, 'DataStoreConnectionHandle')
654 else:
655 return None
656 finally:
657 if self.tracking:
658 self.run_command('disableDataTracking')
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500659
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500660 def build_file(self, buildfile, task, internal=True):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500661 """
662 Runs the specified task for just a single recipe (i.e. no dependencies).
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500663 This is equivalent to bitbake -b, except with the default internal=True
664 no warning about dependencies will be produced, normal info messages
665 from the runqueue will be silenced and BuildInit, BuildStarted and
666 BuildCompleted events will not be fired.
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500667 """
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500668 return self.run_command('buildFile', buildfile, task, internal)
669
670 def build_targets(self, targets, task=None, handle_events=True, extra_events=None, event_callback=None):
671 """
672 Builds the specified targets. This is equivalent to a normal invocation
673 of bitbake. Has built-in event handling which is enabled by default and
674 can be extended if needed.
675 Parameters:
676 targets:
677 One or more targets to build. Can be a list or a
678 space-separated string.
679 task:
680 The task to run; if None then the value of BB_DEFAULT_TASK
681 will be used. Default None.
682 handle_events:
683 True to handle events in a similar way to normal bitbake
684 invocation with knotty; False to return immediately (on the
685 assumption that the caller will handle the events instead).
686 Default True.
687 extra_events:
688 An optional list of events to add to the event mask (if
689 handle_events=True). If you add events here you also need
690 to specify a callback function in event_callback that will
691 handle the additional events. Default None.
692 event_callback:
693 An optional function taking a single parameter which
694 will be called first upon receiving any event (if
695 handle_events=True) so that the caller can override or
696 extend the event handling. Default None.
697 """
698 if isinstance(targets, str):
699 targets = targets.split()
700 if not task:
701 task = self.config_data.getVar('BB_DEFAULT_TASK')
702
703 if handle_events:
704 # A reasonable set of default events matching up with those we handle below
705 eventmask = [
706 'bb.event.BuildStarted',
707 'bb.event.BuildCompleted',
708 'logging.LogRecord',
709 'bb.event.NoProvider',
710 'bb.command.CommandCompleted',
711 'bb.command.CommandFailed',
712 'bb.build.TaskStarted',
713 'bb.build.TaskFailed',
714 'bb.build.TaskSucceeded',
715 'bb.build.TaskFailedSilent',
716 'bb.build.TaskProgress',
717 'bb.runqueue.runQueueTaskStarted',
718 'bb.runqueue.sceneQueueTaskStarted',
719 'bb.event.ProcessStarted',
720 'bb.event.ProcessProgress',
721 'bb.event.ProcessFinished',
722 ]
723 if extra_events:
724 eventmask.extend(extra_events)
725 ret = self.set_event_mask(eventmask)
726
727 includelogs = self.config_data.getVar('BBINCLUDELOGS')
728 loglines = self.config_data.getVar('BBINCLUDELOGS_LINES')
729
730 ret = self.run_command('buildTargets', targets, task)
731 if handle_events:
732 result = False
733 # Borrowed from knotty, instead somewhat hackily we use the helper
734 # as the object to store "shutdown" on
735 helper = bb.ui.uihelper.BBUIHelper()
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500736 helper.shutdown = 0
737 parseprogress = None
Andrew Geissler82c905d2020-04-13 13:39:40 -0500738 termfilter = bb.ui.knotty.TerminalFilter(helper, helper, self.logger.handlers, quiet=self.quiet)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500739 try:
740 while True:
741 try:
742 event = self.wait_event(0.25)
743 if event:
744 if event_callback and event_callback(event):
745 continue
746 if helper.eventHandler(event):
747 if isinstance(event, bb.build.TaskFailedSilent):
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500748 self.logger.warning("Logfile for failed setscene task is %s" % event.logfile)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500749 elif isinstance(event, bb.build.TaskFailed):
750 bb.ui.knotty.print_event_log(event, includelogs, loglines, termfilter)
751 continue
752 if isinstance(event, bb.event.ProcessStarted):
753 if self.quiet > 1:
754 continue
755 parseprogress = bb.ui.knotty.new_progress(event.processname, event.total)
756 parseprogress.start(False)
757 continue
758 if isinstance(event, bb.event.ProcessProgress):
759 if self.quiet > 1:
760 continue
761 if parseprogress:
762 parseprogress.update(event.progress)
763 else:
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000764 bb.warn("Got ProcessProgress event for something that never started?")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500765 continue
766 if isinstance(event, bb.event.ProcessFinished):
767 if self.quiet > 1:
768 continue
769 if parseprogress:
770 parseprogress.finish()
771 parseprogress = None
772 continue
773 if isinstance(event, bb.command.CommandCompleted):
774 result = True
775 break
776 if isinstance(event, bb.command.CommandFailed):
777 self.logger.error(str(event))
778 result = False
779 break
780 if isinstance(event, logging.LogRecord):
781 if event.taskpid == 0 or event.levelno > logging.INFO:
782 self.logger.handle(event)
783 continue
784 if isinstance(event, bb.event.NoProvider):
785 self.logger.error(str(event))
786 result = False
787 break
788
789 elif helper.shutdown > 1:
790 break
791 termfilter.updateFooter()
792 except KeyboardInterrupt:
793 termfilter.clearFooter()
794 if helper.shutdown == 1:
795 print("\nSecond Keyboard Interrupt, stopping...\n")
796 ret = self.run_command("stateForceShutdown")
797 if ret and ret[2]:
798 self.logger.error("Unable to cleanly stop: %s" % ret[2])
799 elif helper.shutdown == 0:
800 print("\nKeyboard Interrupt, closing down...\n")
801 interrupted = True
802 ret = self.run_command("stateShutdown")
803 if ret and ret[2]:
804 self.logger.error("Unable to cleanly shutdown: %s" % ret[2])
805 helper.shutdown = helper.shutdown + 1
806 termfilter.clearFooter()
807 finally:
808 termfilter.finish()
809 if helper.failed_tasks:
810 result = False
811 return result
812 else:
813 return ret
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600814
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500815 def shutdown(self):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500816 """
817 Shut down tinfoil. Disconnects from the server and gracefully
818 releases any associated resources. You must call this function if
819 prepare() has been called, or use a with... block when you create
820 the tinfoil object which will ensure that it gets called.
821 """
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500822 try:
823 if self.server_connection:
824 try:
825 self.run_command('clientComplete')
826 finally:
827 _server_connections.remove(self.server_connection)
828 bb.event.ui_queue = []
829 self.server_connection.terminate()
830 self.server_connection = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500831
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500832 finally:
833 # Restore logging handlers to how it looked when we started
834 if self.oldhandlers:
835 for handler in self.logger.handlers:
836 if handler not in self.oldhandlers:
837 self.logger.handlers.remove(handler)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500838
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500839 def _reconvert_type(self, obj, origtypename):
840 """
841 Convert an object back to the right type, in the case
842 that marshalling has changed it (especially with xmlrpc)
843 """
844 supported_types = {
845 'set': set,
846 'DataStoreConnectionHandle': bb.command.DataStoreConnectionHandle,
847 }
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500848
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500849 origtype = supported_types.get(origtypename, None)
850 if origtype is None:
851 raise Exception('Unsupported type "%s"' % origtypename)
852 if type(obj) == origtype:
853 newobj = obj
854 elif isinstance(obj, dict):
855 # New style class
856 newobj = origtype()
857 for k,v in obj.items():
858 setattr(newobj, k, v)
859 else:
860 # Assume we can coerce the type
861 newobj = origtype(obj)
862
863 if isinstance(newobj, bb.command.DataStoreConnectionHandle):
Andrew Geissler82c905d2020-04-13 13:39:40 -0500864 newobj = TinfoilDataStoreConnector(self, newobj.dsindex)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500865
866 return newobj
867
868
869class TinfoilConfigParameters(BitBakeConfigParameters):
870
871 def __init__(self, config_only, **options):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500872 self.initial_options = options
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500873 # Apply some sane defaults
874 if not 'parse_only' in options:
875 self.initial_options['parse_only'] = not config_only
876 #if not 'status_only' in options:
877 # self.initial_options['status_only'] = config_only
878 if not 'ui' in options:
879 self.initial_options['ui'] = 'knotty'
880 if not 'argv' in options:
881 self.initial_options['argv'] = []
882
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500883 super(TinfoilConfigParameters, self).__init__()
884
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500885 def parseCommandLine(self, argv=None):
886 # We don't want any parameters parsed from the command line
887 opts = super(TinfoilConfigParameters, self).parseCommandLine([])
888 for key, val in self.initial_options.items():
889 setattr(opts[0], key, val)
890 return opts