blob: fa95f6329f89a961487053c4f66d10596769fd1b [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# tinfoil: a simple wrapper around cooker for bitbake-based command-line utilities
2#
Brad Bishop6e60e8b2018-02-01 10:27:11 -05003# Copyright (C) 2012-2017 Intel Corporation
Patrick Williamsc124f4f2015-09-15 14:41:29 -05004# Copyright (C) 2011 Mentor Graphics Corporation
Brad Bishopd7bf8c12018-02-25 22:55:05 -05005# Copyright (C) 2006-2012 Richard Purdie
Patrick Williamsc124f4f2015-09-15 14:41:29 -05006#
7# This program is free software; you can redistribute it and/or modify
8# it under the terms of the GNU General Public License version 2 as
9# published by the Free Software Foundation.
10#
11# This program is distributed in the hope that it will be useful,
12# but WITHOUT ANY WARRANTY; without even the implied warranty of
13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14# GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License along
17# with this program; if not, write to the Free Software Foundation, Inc.,
18# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20import logging
Patrick Williamsc124f4f2015-09-15 14:41:29 -050021import os
22import sys
Brad Bishop6e60e8b2018-02-01 10:27:11 -050023import atexit
24import re
25from collections import OrderedDict, defaultdict
Patrick Williamsc124f4f2015-09-15 14:41:29 -050026
27import bb.cache
28import bb.cooker
29import bb.providers
Brad Bishop6e60e8b2018-02-01 10:27:11 -050030import bb.taskdata
Patrick Williamsc124f4f2015-09-15 14:41:29 -050031import bb.utils
Brad Bishop6e60e8b2018-02-01 10:27:11 -050032import bb.command
33import bb.remotedata
Patrick Williamsc124f4f2015-09-15 14:41:29 -050034from bb.cookerdata import CookerConfiguration, ConfigParameters
Brad Bishop6e60e8b2018-02-01 10:27:11 -050035from bb.main import setup_bitbake, BitBakeConfigParameters, BBMainException
Patrick Williamsc124f4f2015-09-15 14:41:29 -050036import bb.fetch2
37
Brad Bishop6e60e8b2018-02-01 10:27:11 -050038
39# We need this in order to shut down the connection to the bitbake server,
40# otherwise the process will never properly exit
41_server_connections = []
42def _terminate_connections():
43 for connection in _server_connections:
44 connection.terminate()
45atexit.register(_terminate_connections)
46
47class TinfoilUIException(Exception):
48 """Exception raised when the UI returns non-zero from its main function"""
49 def __init__(self, returncode):
50 self.returncode = returncode
51 def __repr__(self):
52 return 'UI module main returned %d' % self.returncode
53
54class TinfoilCommandFailed(Exception):
55 """Exception raised when run_command fails"""
56
57class TinfoilDataStoreConnector:
Brad Bishopd7bf8c12018-02-25 22:55:05 -050058 """Connector object used to enable access to datastore objects via tinfoil"""
Brad Bishop6e60e8b2018-02-01 10:27:11 -050059
60 def __init__(self, tinfoil, dsindex):
61 self.tinfoil = tinfoil
62 self.dsindex = dsindex
63 def getVar(self, name):
64 value = self.tinfoil.run_command('dataStoreConnectorFindVar', self.dsindex, name)
65 overrides = None
66 if isinstance(value, dict):
67 if '_connector_origtype' in value:
68 value['_content'] = self.tinfoil._reconvert_type(value['_content'], value['_connector_origtype'])
69 del value['_connector_origtype']
70 if '_connector_overrides' in value:
71 overrides = value['_connector_overrides']
72 del value['_connector_overrides']
73 return value, overrides
74 def getKeys(self):
75 return set(self.tinfoil.run_command('dataStoreConnectorGetKeys', self.dsindex))
76 def getVarHistory(self, name):
77 return self.tinfoil.run_command('dataStoreConnectorGetVarHistory', self.dsindex, name)
78 def expandPythonRef(self, varname, expr, d):
79 ds = bb.remotedata.RemoteDatastores.transmit_datastore(d)
80 ret = self.tinfoil.run_command('dataStoreConnectorExpandPythonRef', ds, varname, expr)
81 return ret
82 def setVar(self, varname, value):
83 if self.dsindex is None:
84 self.tinfoil.run_command('setVariable', varname, value)
85 else:
86 # Not currently implemented - indicate that setting should
87 # be redirected to local side
88 return True
89 def setVarFlag(self, varname, flagname, value):
90 if self.dsindex is None:
91 self.tinfoil.run_command('dataStoreConnectorSetVarFlag', self.dsindex, varname, flagname, value)
92 else:
93 # Not currently implemented - indicate that setting should
94 # be redirected to local side
95 return True
96 def delVar(self, varname):
97 if self.dsindex is None:
98 self.tinfoil.run_command('dataStoreConnectorDelVar', self.dsindex, varname)
99 else:
100 # Not currently implemented - indicate that setting should
101 # be redirected to local side
102 return True
103 def delVarFlag(self, varname, flagname):
104 if self.dsindex is None:
105 self.tinfoil.run_command('dataStoreConnectorDelVar', self.dsindex, varname, flagname)
106 else:
107 # Not currently implemented - indicate that setting should
108 # be redirected to local side
109 return True
110 def renameVar(self, name, newname):
111 if self.dsindex is None:
112 self.tinfoil.run_command('dataStoreConnectorRenameVar', self.dsindex, name, newname)
113 else:
114 # Not currently implemented - indicate that setting should
115 # be redirected to local side
116 return True
117
118class TinfoilCookerAdapter:
119 """
120 Provide an adapter for existing code that expects to access a cooker object via Tinfoil,
121 since now Tinfoil is on the client side it no longer has direct access.
122 """
123
124 class TinfoilCookerCollectionAdapter:
125 """ cooker.collection adapter """
126 def __init__(self, tinfoil):
127 self.tinfoil = tinfoil
128 def get_file_appends(self, fn):
129 return self.tinfoil.get_file_appends(fn)
130 def __getattr__(self, name):
131 if name == 'overlayed':
132 return self.tinfoil.get_overlayed_recipes()
133 elif name == 'bbappends':
134 return self.tinfoil.run_command('getAllAppends')
135 else:
136 raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
137
138 class TinfoilRecipeCacheAdapter:
139 """ cooker.recipecache adapter """
140 def __init__(self, tinfoil):
141 self.tinfoil = tinfoil
142 self._cache = {}
143
144 def get_pkg_pn_fn(self):
145 pkg_pn = defaultdict(list, self.tinfoil.run_command('getRecipes') or [])
146 pkg_fn = {}
147 for pn, fnlist in pkg_pn.items():
148 for fn in fnlist:
149 pkg_fn[fn] = pn
150 self._cache['pkg_pn'] = pkg_pn
151 self._cache['pkg_fn'] = pkg_fn
152
153 def __getattr__(self, name):
154 # Grab these only when they are requested since they aren't always used
155 if name in self._cache:
156 return self._cache[name]
157 elif name == 'pkg_pn':
158 self.get_pkg_pn_fn()
159 return self._cache[name]
160 elif name == 'pkg_fn':
161 self.get_pkg_pn_fn()
162 return self._cache[name]
163 elif name == 'deps':
164 attrvalue = defaultdict(list, self.tinfoil.run_command('getRecipeDepends') or [])
165 elif name == 'rundeps':
166 attrvalue = defaultdict(lambda: defaultdict(list), self.tinfoil.run_command('getRuntimeDepends') or [])
167 elif name == 'runrecs':
168 attrvalue = defaultdict(lambda: defaultdict(list), self.tinfoil.run_command('getRuntimeRecommends') or [])
169 elif name == 'pkg_pepvpr':
170 attrvalue = self.tinfoil.run_command('getRecipeVersions') or {}
171 elif name == 'inherits':
172 attrvalue = self.tinfoil.run_command('getRecipeInherits') or {}
173 elif name == 'bbfile_priority':
174 attrvalue = self.tinfoil.run_command('getBbFilePriority') or {}
175 elif name == 'pkg_dp':
176 attrvalue = self.tinfoil.run_command('getDefaultPreference') or {}
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500177 elif name == 'fn_provides':
178 attrvalue = self.tinfoil.run_command('getRecipeProvides') or {}
179 elif name == 'packages':
180 attrvalue = self.tinfoil.run_command('getRecipePackages') or {}
181 elif name == 'packages_dynamic':
182 attrvalue = self.tinfoil.run_command('getRecipePackagesDynamic') or {}
183 elif name == 'rproviders':
184 attrvalue = self.tinfoil.run_command('getRProviders') or {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500185 else:
186 raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
187
188 self._cache[name] = attrvalue
189 return attrvalue
190
191 def __init__(self, tinfoil):
192 self.tinfoil = tinfoil
193 self.collection = self.TinfoilCookerCollectionAdapter(tinfoil)
194 self.recipecaches = {}
195 # FIXME all machines
196 self.recipecaches[''] = self.TinfoilRecipeCacheAdapter(tinfoil)
197 self._cache = {}
198 def __getattr__(self, name):
199 # Grab these only when they are requested since they aren't always used
200 if name in self._cache:
201 return self._cache[name]
202 elif name == 'skiplist':
203 attrvalue = self.tinfoil.get_skipped_recipes()
204 elif name == 'bbfile_config_priorities':
205 ret = self.tinfoil.run_command('getLayerPriorities')
206 bbfile_config_priorities = []
207 for collection, pattern, regex, pri in ret:
208 bbfile_config_priorities.append((collection, pattern, re.compile(regex), pri))
209
210 attrvalue = bbfile_config_priorities
211 else:
212 raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
213
214 self._cache[name] = attrvalue
215 return attrvalue
216
217 def findBestProvider(self, pn):
218 return self.tinfoil.find_best_provider(pn)
219
220
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500221class TinfoilRecipeInfo:
222 """
223 Provides a convenient representation of the cached information for a single recipe.
224 Some attributes are set on construction, others are read on-demand (which internally
225 may result in a remote procedure call to the bitbake server the first time).
226 Note that only information which is cached is available through this object - if
227 you need other variable values you will need to parse the recipe using
228 Tinfoil.parse_recipe().
229 """
230 def __init__(self, recipecache, d, pn, fn, fns):
231 self._recipecache = recipecache
232 self._d = d
233 self.pn = pn
234 self.fn = fn
235 self.fns = fns
236 self.inherit_files = recipecache.inherits[fn]
237 self.depends = recipecache.deps[fn]
238 (self.pe, self.pv, self.pr) = recipecache.pkg_pepvpr[fn]
239 self._cached_packages = None
240 self._cached_rprovides = None
241 self._cached_packages_dynamic = None
242
243 def __getattr__(self, name):
244 if name == 'alternates':
245 return [x for x in self.fns if x != self.fn]
246 elif name == 'rdepends':
247 return self._recipecache.rundeps[self.fn]
248 elif name == 'rrecommends':
249 return self._recipecache.runrecs[self.fn]
250 elif name == 'provides':
251 return self._recipecache.fn_provides[self.fn]
252 elif name == 'packages':
253 if self._cached_packages is None:
254 self._cached_packages = []
255 for pkg, fns in self._recipecache.packages.items():
256 if self.fn in fns:
257 self._cached_packages.append(pkg)
258 return self._cached_packages
259 elif name == 'packages_dynamic':
260 if self._cached_packages_dynamic is None:
261 self._cached_packages_dynamic = []
262 for pkg, fns in self._recipecache.packages_dynamic.items():
263 if self.fn in fns:
264 self._cached_packages_dynamic.append(pkg)
265 return self._cached_packages_dynamic
266 elif name == 'rprovides':
267 if self._cached_rprovides is None:
268 self._cached_rprovides = []
269 for pkg, fns in self._recipecache.rproviders.items():
270 if self.fn in fns:
271 self._cached_rprovides.append(pkg)
272 return self._cached_rprovides
273 else:
274 raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
275 def inherits(self, only_recipe=False):
276 """
277 Get the inherited classes for a recipe. Returns the class names only.
278 Parameters:
279 only_recipe: True to return only the classes inherited by the recipe
280 itself, False to return all classes inherited within
281 the context for the recipe (which includes globally
282 inherited classes).
283 """
284 if only_recipe:
285 global_inherit = [x for x in (self._d.getVar('BBINCLUDED') or '').split() if x.endswith('.bbclass')]
286 else:
287 global_inherit = []
288 for clsfile in self.inherit_files:
289 if only_recipe and clsfile in global_inherit:
290 continue
291 clsname = os.path.splitext(os.path.basename(clsfile))[0]
292 yield clsname
293 def __str__(self):
294 return '%s' % self.pn
295
296
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500297class Tinfoil:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500298 """
299 Tinfoil - an API for scripts and utilities to query
300 BitBake internals and perform build operations.
301 """
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500302
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500303 def __init__(self, output=sys.stdout, tracking=False, setup_logging=True):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500304 """
305 Create a new tinfoil object.
306 Parameters:
307 output: specifies where console output should be sent. Defaults
308 to sys.stdout.
309 tracking: True to enable variable history tracking, False to
310 disable it (default). Enabling this has a minor
311 performance impact so typically it isn't enabled
312 unless you need to query variable history.
313 setup_logging: True to setup a logger so that things like
314 bb.warn() will work immediately and timeout warnings
315 are visible; False to let BitBake do this itself.
316 """
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500317 self.logger = logging.getLogger('BitBake')
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500318 self.config_data = None
319 self.cooker = None
320 self.tracking = tracking
321 self.ui_module = None
322 self.server_connection = None
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500323 self.recipes_parsed = False
324 self.quiet = 0
325 self.oldhandlers = self.logger.handlers[:]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500326 if setup_logging:
327 # This is the *client-side* logger, nothing to do with
328 # logging messages from the server
329 bb.msg.logger_create('BitBake', output)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500330 self.localhandlers = []
331 for handler in self.logger.handlers:
332 if handler not in self.oldhandlers:
333 self.localhandlers.append(handler)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500334
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600335 def __enter__(self):
336 return self
337
338 def __exit__(self, type, value, traceback):
339 self.shutdown()
340
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500341 def prepare(self, config_only=False, config_params=None, quiet=0, extra_features=None):
342 """
343 Prepares the underlying BitBake system to be used via tinfoil.
344 This function must be called prior to calling any of the other
345 functions in the API.
346 NOTE: if you call prepare() you must absolutely call shutdown()
347 before your code terminates. You can use a "with" block to ensure
348 this happens e.g.
349
350 with bb.tinfoil.Tinfoil() as tinfoil:
351 tinfoil.prepare()
352 ...
353
354 Parameters:
355 config_only: True to read only the configuration and not load
356 the cache / parse recipes. This is useful if you just
357 want to query the value of a variable at the global
358 level or you want to do anything else that doesn't
359 involve knowing anything about the recipes in the
360 current configuration. False loads the cache / parses
361 recipes.
362 config_params: optionally specify your own configuration
363 parameters. If not specified an instance of
364 TinfoilConfigParameters will be created internally.
365 quiet: quiet level controlling console output - equivalent
366 to bitbake's -q/--quiet option. Default of 0 gives
367 the same output level as normal bitbake execution.
368 extra_features: extra features to be added to the feature
369 set requested from the server. See
370 CookerFeatures._feature_list for possible
371 features.
372 """
373 self.quiet = quiet
374
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500375 if self.tracking:
376 extrafeatures = [bb.cooker.CookerFeatures.BASEDATASTORE_TRACKING]
377 else:
378 extrafeatures = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500379
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500380 if extra_features:
381 extrafeatures += extra_features
382
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500383 if not config_params:
384 config_params = TinfoilConfigParameters(config_only=config_only, quiet=quiet)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500385
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500386 cookerconfig = CookerConfiguration()
387 cookerconfig.setConfigParameters(config_params)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500388
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500389 if not config_only:
390 # Disable local loggers because the UI module is going to set up its own
391 for handler in self.localhandlers:
392 self.logger.handlers.remove(handler)
393 self.localhandlers = []
394
395 self.server_connection, ui_module = setup_bitbake(config_params,
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500396 cookerconfig,
397 extrafeatures)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500398
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500399 self.ui_module = ui_module
400
401 # Ensure the path to bitbake's bin directory is in PATH so that things like
402 # bitbake-worker can be run (usually this is the case, but it doesn't have to be)
403 path = os.getenv('PATH').split(':')
404 bitbakebinpath = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', 'bin'))
405 for entry in path:
406 if entry.endswith(os.sep):
407 entry = entry[:-1]
408 if os.path.abspath(entry) == bitbakebinpath:
409 break
410 else:
411 path.insert(0, bitbakebinpath)
412 os.environ['PATH'] = ':'.join(path)
413
414 if self.server_connection:
415 _server_connections.append(self.server_connection)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500416 if config_only:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500417 config_params.updateToServer(self.server_connection.connection, os.environ.copy())
418 self.run_command('parseConfiguration')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500419 else:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500420 self.run_actions(config_params)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500421 self.recipes_parsed = True
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500422
423 self.config_data = bb.data.init()
424 connector = TinfoilDataStoreConnector(self, None)
425 self.config_data.setVar('_remote_data', connector)
426 self.cooker = TinfoilCookerAdapter(self)
427 self.cooker_data = self.cooker.recipecaches['']
428 else:
429 raise Exception('Failed to start bitbake server')
430
431 def run_actions(self, config_params):
432 """
433 Run the actions specified in config_params through the UI.
434 """
435 ret = self.ui_module.main(self.server_connection.connection, self.server_connection.events, config_params)
436 if ret:
437 raise TinfoilUIException(ret)
438
439 def parseRecipes(self):
440 """
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500441 Legacy function - use parse_recipes() instead.
442 """
443 self.parse_recipes()
444
445 def parse_recipes(self):
446 """
447 Load information on all recipes. Normally you should specify
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500448 config_only=False when calling prepare() instead of using this
449 function; this function is designed for situations where you need
450 to initialise Tinfoil and use it with config_only=True first and
451 then conditionally call this function to parse recipes later.
452 """
453 config_params = TinfoilConfigParameters(config_only=False)
454 self.run_actions(config_params)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500455 self.recipes_parsed = True
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500456
457 def run_command(self, command, *params):
458 """
459 Run a command on the server (as implemented in bb.command).
460 Note that there are two types of command - synchronous and
461 asynchronous; in order to receive the results of asynchronous
462 commands you will need to set an appropriate event mask
463 using set_event_mask() and listen for the result using
464 wait_event() - with the correct event mask you'll at least get
465 bb.command.CommandCompleted and possibly other events before
466 that depending on the command.
467 """
468 if not self.server_connection:
469 raise Exception('Not connected to server (did you call .prepare()?)')
470
471 commandline = [command]
472 if params:
473 commandline.extend(params)
474 result = self.server_connection.connection.runCommand(commandline)
475 if result[1]:
476 raise TinfoilCommandFailed(result[1])
477 return result[0]
478
479 def set_event_mask(self, eventlist):
480 """Set the event mask which will be applied within wait_event()"""
481 if not self.server_connection:
482 raise Exception('Not connected to server (did you call .prepare()?)')
483 llevel, debug_domains = bb.msg.constructLogOptions()
484 ret = self.run_command('setEventMask', self.server_connection.connection.getEventHandle(), llevel, debug_domains, eventlist)
485 if not ret:
486 raise Exception('setEventMask failed')
487
488 def wait_event(self, timeout=0):
489 """
490 Wait for an event from the server for the specified time.
491 A timeout of 0 means don't wait if there are no events in the queue.
492 Returns the next event in the queue or None if the timeout was
493 reached. Note that in order to recieve any events you will
494 first need to set the internal event mask using set_event_mask()
495 (otherwise whatever event mask the UI set up will be in effect).
496 """
497 if not self.server_connection:
498 raise Exception('Not connected to server (did you call .prepare()?)')
499 return self.server_connection.events.waitEvent(timeout)
500
501 def get_overlayed_recipes(self):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500502 """
503 Find recipes which are overlayed (i.e. where recipes exist in multiple layers)
504 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500505 return defaultdict(list, self.run_command('getOverlayedRecipes'))
506
507 def get_skipped_recipes(self):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500508 """
509 Find recipes which were skipped (i.e. SkipRecipe was raised
510 during parsing).
511 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500512 return OrderedDict(self.run_command('getSkippedRecipes'))
513
514 def get_all_providers(self):
515 return defaultdict(list, self.run_command('allProviders'))
516
517 def find_providers(self):
518 return self.run_command('findProviders')
519
520 def find_best_provider(self, pn):
521 return self.run_command('findBestProvider', pn)
522
523 def get_runtime_providers(self, rdep):
524 return self.run_command('getRuntimeProviders', rdep)
525
526 def get_recipe_file(self, pn):
527 """
528 Get the file name for the specified recipe/target. Raises
529 bb.providers.NoProvider if there is no match or the recipe was
530 skipped.
531 """
532 best = self.find_best_provider(pn)
533 if not best or (len(best) > 3 and not best[3]):
534 skiplist = self.get_skipped_recipes()
535 taskdata = bb.taskdata.TaskData(None, skiplist=skiplist)
536 skipreasons = taskdata.get_reasons(pn)
537 if skipreasons:
538 raise bb.providers.NoProvider('%s is unavailable:\n %s' % (pn, ' \n'.join(skipreasons)))
539 else:
540 raise bb.providers.NoProvider('Unable to find any recipe file matching "%s"' % pn)
541 return best[3]
542
543 def get_file_appends(self, fn):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500544 """
545 Find the bbappends for a recipe file
546 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500547 return self.run_command('getFileAppends', fn)
548
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500549 def all_recipes(self, mc='', sort=True):
550 """
551 Enable iterating over all recipes in the current configuration.
552 Returns an iterator over TinfoilRecipeInfo objects created on demand.
553 Parameters:
554 mc: The multiconfig, default of '' uses the main configuration.
555 sort: True to sort recipes alphabetically (default), False otherwise
556 """
557 recipecache = self.cooker.recipecaches[mc]
558 if sort:
559 recipes = sorted(recipecache.pkg_pn.items())
560 else:
561 recipes = recipecache.pkg_pn.items()
562 for pn, fns in recipes:
563 prov = self.find_best_provider(pn)
564 recipe = TinfoilRecipeInfo(recipecache,
565 self.config_data,
566 pn=pn,
567 fn=prov[3],
568 fns=fns)
569 yield recipe
570
571 def all_recipe_files(self, mc='', variants=True, preferred_only=False):
572 """
573 Enable iterating over all recipe files in the current configuration.
574 Returns an iterator over file paths.
575 Parameters:
576 mc: The multiconfig, default of '' uses the main configuration.
577 variants: True to include variants of recipes created through
578 BBCLASSEXTEND (default) or False to exclude them
579 preferred_only: True to include only the preferred recipe where
580 multiple exist providing the same PN, False to list
581 all recipes
582 """
583 recipecache = self.cooker.recipecaches[mc]
584 if preferred_only:
585 files = []
586 for pn in recipecache.pkg_pn.keys():
587 prov = self.find_best_provider(pn)
588 files.append(prov[3])
589 else:
590 files = recipecache.pkg_fn.keys()
591 for fn in sorted(files):
592 if not variants and fn.startswith('virtual:'):
593 continue
594 yield fn
595
596
597 def get_recipe_info(self, pn, mc=''):
598 """
599 Get information on a specific recipe in the current configuration by name (PN).
600 Returns a TinfoilRecipeInfo object created on demand.
601 Parameters:
602 mc: The multiconfig, default of '' uses the main configuration.
603 """
604 recipecache = self.cooker.recipecaches[mc]
605 prov = self.find_best_provider(pn)
606 fn = prov[3]
607 actual_pn = recipecache.pkg_fn[fn]
608 recipe = TinfoilRecipeInfo(recipecache,
609 self.config_data,
610 pn=actual_pn,
611 fn=fn,
612 fns=recipecache.pkg_pn[actual_pn])
613 return recipe
614
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500615 def parse_recipe(self, pn):
616 """
617 Parse the specified recipe and return a datastore object
618 representing the environment for the recipe.
619 """
620 fn = self.get_recipe_file(pn)
621 return self.parse_recipe_file(fn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500622
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600623 def parse_recipe_file(self, fn, appends=True, appendlist=None, config_data=None):
624 """
625 Parse the specified recipe file (with or without bbappends)
626 and return a datastore object representing the environment
627 for the recipe.
628 Parameters:
629 fn: recipe file to parse - can be a file path or virtual
630 specification
631 appends: True to apply bbappends, False otherwise
632 appendlist: optional list of bbappend files to apply, if you
633 want to filter them
634 config_data: custom config datastore to use. NOTE: if you
635 specify config_data then you cannot use a virtual
636 specification for fn.
637 """
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500638 if self.tracking:
639 # Enable history tracking just for the parse operation
640 self.run_command('enableDataTracking')
641 try:
642 if appends and appendlist == []:
643 appends = False
644 if config_data:
645 dctr = bb.remotedata.RemoteDatastores.transmit_datastore(config_data)
646 dscon = self.run_command('parseRecipeFile', fn, appends, appendlist, dctr)
647 else:
648 dscon = self.run_command('parseRecipeFile', fn, appends, appendlist)
649 if dscon:
650 return self._reconvert_type(dscon, 'DataStoreConnectionHandle')
651 else:
652 return None
653 finally:
654 if self.tracking:
655 self.run_command('disableDataTracking')
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500656
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500657 def build_file(self, buildfile, task, internal=True):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500658 """
659 Runs the specified task for just a single recipe (i.e. no dependencies).
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500660 This is equivalent to bitbake -b, except with the default internal=True
661 no warning about dependencies will be produced, normal info messages
662 from the runqueue will be silenced and BuildInit, BuildStarted and
663 BuildCompleted events will not be fired.
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500664 """
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500665 return self.run_command('buildFile', buildfile, task, internal)
666
667 def build_targets(self, targets, task=None, handle_events=True, extra_events=None, event_callback=None):
668 """
669 Builds the specified targets. This is equivalent to a normal invocation
670 of bitbake. Has built-in event handling which is enabled by default and
671 can be extended if needed.
672 Parameters:
673 targets:
674 One or more targets to build. Can be a list or a
675 space-separated string.
676 task:
677 The task to run; if None then the value of BB_DEFAULT_TASK
678 will be used. Default None.
679 handle_events:
680 True to handle events in a similar way to normal bitbake
681 invocation with knotty; False to return immediately (on the
682 assumption that the caller will handle the events instead).
683 Default True.
684 extra_events:
685 An optional list of events to add to the event mask (if
686 handle_events=True). If you add events here you also need
687 to specify a callback function in event_callback that will
688 handle the additional events. Default None.
689 event_callback:
690 An optional function taking a single parameter which
691 will be called first upon receiving any event (if
692 handle_events=True) so that the caller can override or
693 extend the event handling. Default None.
694 """
695 if isinstance(targets, str):
696 targets = targets.split()
697 if not task:
698 task = self.config_data.getVar('BB_DEFAULT_TASK')
699
700 if handle_events:
701 # A reasonable set of default events matching up with those we handle below
702 eventmask = [
703 'bb.event.BuildStarted',
704 'bb.event.BuildCompleted',
705 'logging.LogRecord',
706 'bb.event.NoProvider',
707 'bb.command.CommandCompleted',
708 'bb.command.CommandFailed',
709 'bb.build.TaskStarted',
710 'bb.build.TaskFailed',
711 'bb.build.TaskSucceeded',
712 'bb.build.TaskFailedSilent',
713 'bb.build.TaskProgress',
714 'bb.runqueue.runQueueTaskStarted',
715 'bb.runqueue.sceneQueueTaskStarted',
716 'bb.event.ProcessStarted',
717 'bb.event.ProcessProgress',
718 'bb.event.ProcessFinished',
719 ]
720 if extra_events:
721 eventmask.extend(extra_events)
722 ret = self.set_event_mask(eventmask)
723
724 includelogs = self.config_data.getVar('BBINCLUDELOGS')
725 loglines = self.config_data.getVar('BBINCLUDELOGS_LINES')
726
727 ret = self.run_command('buildTargets', targets, task)
728 if handle_events:
729 result = False
730 # Borrowed from knotty, instead somewhat hackily we use the helper
731 # as the object to store "shutdown" on
732 helper = bb.ui.uihelper.BBUIHelper()
733 # We set up logging optionally in the constructor so now we need to
734 # grab the handlers to pass to TerminalFilter
735 console = None
736 errconsole = None
737 for handler in self.logger.handlers:
738 if isinstance(handler, logging.StreamHandler):
739 if handler.stream == sys.stdout:
740 console = handler
741 elif handler.stream == sys.stderr:
742 errconsole = handler
743 format_str = "%(levelname)s: %(message)s"
744 format = bb.msg.BBLogFormatter(format_str)
745 helper.shutdown = 0
746 parseprogress = None
747 termfilter = bb.ui.knotty.TerminalFilter(helper, helper, console, errconsole, format, quiet=self.quiet)
748 try:
749 while True:
750 try:
751 event = self.wait_event(0.25)
752 if event:
753 if event_callback and event_callback(event):
754 continue
755 if helper.eventHandler(event):
756 if isinstance(event, bb.build.TaskFailedSilent):
757 logger.warning("Logfile for failed setscene task is %s" % event.logfile)
758 elif isinstance(event, bb.build.TaskFailed):
759 bb.ui.knotty.print_event_log(event, includelogs, loglines, termfilter)
760 continue
761 if isinstance(event, bb.event.ProcessStarted):
762 if self.quiet > 1:
763 continue
764 parseprogress = bb.ui.knotty.new_progress(event.processname, event.total)
765 parseprogress.start(False)
766 continue
767 if isinstance(event, bb.event.ProcessProgress):
768 if self.quiet > 1:
769 continue
770 if parseprogress:
771 parseprogress.update(event.progress)
772 else:
773 bb.warn("Got ProcessProgress event for someting that never started?")
774 continue
775 if isinstance(event, bb.event.ProcessFinished):
776 if self.quiet > 1:
777 continue
778 if parseprogress:
779 parseprogress.finish()
780 parseprogress = None
781 continue
782 if isinstance(event, bb.command.CommandCompleted):
783 result = True
784 break
785 if isinstance(event, bb.command.CommandFailed):
786 self.logger.error(str(event))
787 result = False
788 break
789 if isinstance(event, logging.LogRecord):
790 if event.taskpid == 0 or event.levelno > logging.INFO:
791 self.logger.handle(event)
792 continue
793 if isinstance(event, bb.event.NoProvider):
794 self.logger.error(str(event))
795 result = False
796 break
797
798 elif helper.shutdown > 1:
799 break
800 termfilter.updateFooter()
801 except KeyboardInterrupt:
802 termfilter.clearFooter()
803 if helper.shutdown == 1:
804 print("\nSecond Keyboard Interrupt, stopping...\n")
805 ret = self.run_command("stateForceShutdown")
806 if ret and ret[2]:
807 self.logger.error("Unable to cleanly stop: %s" % ret[2])
808 elif helper.shutdown == 0:
809 print("\nKeyboard Interrupt, closing down...\n")
810 interrupted = True
811 ret = self.run_command("stateShutdown")
812 if ret and ret[2]:
813 self.logger.error("Unable to cleanly shutdown: %s" % ret[2])
814 helper.shutdown = helper.shutdown + 1
815 termfilter.clearFooter()
816 finally:
817 termfilter.finish()
818 if helper.failed_tasks:
819 result = False
820 return result
821 else:
822 return ret
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600823
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500824 def shutdown(self):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500825 """
826 Shut down tinfoil. Disconnects from the server and gracefully
827 releases any associated resources. You must call this function if
828 prepare() has been called, or use a with... block when you create
829 the tinfoil object which will ensure that it gets called.
830 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500831 if self.server_connection:
832 self.run_command('clientComplete')
833 _server_connections.remove(self.server_connection)
834 bb.event.ui_queue = []
835 self.server_connection.terminate()
836 self.server_connection = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500837
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500838 # Restore logging handlers to how it looked when we started
839 if self.oldhandlers:
840 for handler in self.logger.handlers:
841 if handler not in self.oldhandlers:
842 self.logger.handlers.remove(handler)
843
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500844 def _reconvert_type(self, obj, origtypename):
845 """
846 Convert an object back to the right type, in the case
847 that marshalling has changed it (especially with xmlrpc)
848 """
849 supported_types = {
850 'set': set,
851 'DataStoreConnectionHandle': bb.command.DataStoreConnectionHandle,
852 }
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500853
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500854 origtype = supported_types.get(origtypename, None)
855 if origtype is None:
856 raise Exception('Unsupported type "%s"' % origtypename)
857 if type(obj) == origtype:
858 newobj = obj
859 elif isinstance(obj, dict):
860 # New style class
861 newobj = origtype()
862 for k,v in obj.items():
863 setattr(newobj, k, v)
864 else:
865 # Assume we can coerce the type
866 newobj = origtype(obj)
867
868 if isinstance(newobj, bb.command.DataStoreConnectionHandle):
869 connector = TinfoilDataStoreConnector(self, newobj.dsindex)
870 newobj = bb.data.init()
871 newobj.setVar('_remote_data', connector)
872
873 return newobj
874
875
876class TinfoilConfigParameters(BitBakeConfigParameters):
877
878 def __init__(self, config_only, **options):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500879 self.initial_options = options
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500880 # Apply some sane defaults
881 if not 'parse_only' in options:
882 self.initial_options['parse_only'] = not config_only
883 #if not 'status_only' in options:
884 # self.initial_options['status_only'] = config_only
885 if not 'ui' in options:
886 self.initial_options['ui'] = 'knotty'
887 if not 'argv' in options:
888 self.initial_options['argv'] = []
889
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500890 super(TinfoilConfigParameters, self).__init__()
891
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500892 def parseCommandLine(self, argv=None):
893 # We don't want any parameters parsed from the command line
894 opts = super(TinfoilConfigParameters, self).parseCommandLine([])
895 for key, val in self.initial_options.items():
896 setattr(opts[0], key, val)
897 return opts