blob: db991702e2d281635a72c4250213bbf66d522dac [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001#
2# Copyright (C) 2003, 2004 Chris Larson
3# Copyright (C) 2003, 2004 Phil Blundell
4# Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
5# Copyright (C) 2005 Holger Hans Peter Freyther
6# Copyright (C) 2005 ROAD GmbH
7# Copyright (C) 2006 - 2007 Richard Purdie
8#
Brad Bishopc342db32019-05-15 21:57:59 -04009# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -050010#
Patrick Williamsc0f7c042017-02-23 20:41:17 -060011
Patrick Williamsc124f4f2015-09-15 14:41:29 -050012import sys, os, glob, os.path, re, time
Patrick Williamsc124f4f2015-09-15 14:41:29 -050013import itertools
14import logging
15import multiprocessing
16import sre_constants
17import threading
Patrick Williamsc0f7c042017-02-23 20:41:17 -060018from io import StringIO, UnsupportedOperation
Patrick Williamsc124f4f2015-09-15 14:41:29 -050019from contextlib import closing
Patrick Williamsc0f7c042017-02-23 20:41:17 -060020from collections import defaultdict, namedtuple
Patrick Williamsc124f4f2015-09-15 14:41:29 -050021import bb, bb.exceptions, bb.command
22from bb import utils, data, parse, event, cache, providers, taskdata, runqueue, build
Patrick Williamsc0f7c042017-02-23 20:41:17 -060023import queue
Patrick Williamsc124f4f2015-09-15 14:41:29 -050024import signal
Patrick Williamsc124f4f2015-09-15 14:41:29 -050025import prserv.serv
26import pyinotify
Patrick Williamsc0f7c042017-02-23 20:41:17 -060027import json
28import pickle
29import codecs
Brad Bishop08902b02019-08-20 09:16:51 -040030import hashserv
Patrick Williamsc124f4f2015-09-15 14:41:29 -050031
32logger = logging.getLogger("BitBake")
33collectlog = logging.getLogger("BitBake.Collection")
34buildlog = logging.getLogger("BitBake.Build")
35parselog = logging.getLogger("BitBake.Parsing")
36providerlog = logging.getLogger("BitBake.Provider")
37
38class NoSpecificMatch(bb.BBHandledException):
39 """
40 Exception raised when no or multiple file matches are found
41 """
42
43class NothingToBuild(Exception):
44 """
45 Exception raised when there is nothing to build
46 """
47
48class CollectionError(bb.BBHandledException):
49 """
50 Exception raised when layer configuration is incorrect
51 """
52
53class state:
Patrick Williamsc0f7c042017-02-23 20:41:17 -060054 initial, parsing, running, shutdown, forceshutdown, stopped, error = list(range(7))
Patrick Williamsc124f4f2015-09-15 14:41:29 -050055
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050056 @classmethod
57 def get_name(cls, code):
58 for name in dir(cls):
59 value = getattr(cls, name)
60 if type(value) == type(cls.initial) and value == code:
61 return name
62 raise ValueError("Invalid status code: %s" % code)
63
Patrick Williamsc124f4f2015-09-15 14:41:29 -050064
65class SkippedPackage:
66 def __init__(self, info = None, reason = None):
67 self.pn = None
68 self.skipreason = None
69 self.provides = None
70 self.rprovides = None
71
72 if info:
73 self.pn = info.pn
74 self.skipreason = info.skipreason
75 self.provides = info.provides
Andrew Geisslerd1e89492021-02-12 15:35:20 -060076 self.rprovides = info.packages + info.rprovides
77 for package in info.packages:
78 self.rprovides += info.rprovides_pkg[package]
Patrick Williamsc124f4f2015-09-15 14:41:29 -050079 elif reason:
80 self.skipreason = reason
81
82
83class CookerFeatures(object):
Patrick Williamsc0f7c042017-02-23 20:41:17 -060084 _feature_list = [HOB_EXTRA_CACHES, BASEDATASTORE_TRACKING, SEND_SANITYEVENTS] = list(range(3))
Patrick Williamsc124f4f2015-09-15 14:41:29 -050085
86 def __init__(self):
87 self._features=set()
88
89 def setFeature(self, f):
90 # validate we got a request for a feature we support
91 if f not in CookerFeatures._feature_list:
92 return
93 self._features.add(f)
94
95 def __contains__(self, f):
96 return f in self._features
97
98 def __iter__(self):
99 return self._features.__iter__()
100
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600101 def __next__(self):
102 return next(self._features)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500103
104
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600105class EventWriter:
106 def __init__(self, cooker, eventfile):
107 self.file_inited = None
108 self.cooker = cooker
109 self.eventfile = eventfile
110 self.event_queue = []
111
112 def write_event(self, event):
113 with open(self.eventfile, "a") as f:
114 try:
115 str_event = codecs.encode(pickle.dumps(event), 'base64').decode('utf-8')
116 f.write("%s\n" % json.dumps({"class": event.__module__ + "." + event.__class__.__name__,
117 "vars": str_event}))
118 except Exception as err:
119 import traceback
120 print(err, traceback.format_exc())
121
122 def send(self, event):
123 if self.file_inited:
124 # we have the file, just write the event
125 self.write_event(event)
126 else:
127 # init on bb.event.BuildStarted
128 name = "%s.%s" % (event.__module__, event.__class__.__name__)
129 if name in ("bb.event.BuildStarted", "bb.cooker.CookerExit"):
130 with open(self.eventfile, "w") as f:
131 f.write("%s\n" % json.dumps({ "allvariables" : self.cooker.getAllKeysWithFlags(["doc", "func"])}))
132
133 self.file_inited = True
134
135 # write pending events
136 for evt in self.event_queue:
137 self.write_event(evt)
138
139 # also write the current event
140 self.write_event(event)
141 else:
142 # queue all events until the file is inited
143 self.event_queue.append(event)
144
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500145#============================================================================#
146# BBCooker
147#============================================================================#
148class BBCooker:
149 """
150 Manages one bitbake build run
151 """
152
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500153 def __init__(self, featureSet=None, idleCallBackRegister=None):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600154 self.recipecaches = None
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500155 self.eventlog = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500156 self.skiplist = {}
157 self.featureset = CookerFeatures()
158 if featureSet:
159 for f in featureSet:
160 self.featureset.setFeature(f)
161
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500162 self.configuration = bb.cookerdata.CookerConfiguration()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500163
Andrew Geissler635e0e42020-08-21 15:58:33 -0500164 self.idleCallBackRegister = idleCallBackRegister
165
Brad Bishopf058f492019-01-28 23:50:33 -0500166 bb.debug(1, "BBCooker starting %s" % time.time())
167 sys.stdout.flush()
168
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500169 self.configwatcher = pyinotify.WatchManager()
Brad Bishopf058f492019-01-28 23:50:33 -0500170 bb.debug(1, "BBCooker pyinotify1 %s" % time.time())
171 sys.stdout.flush()
172
Andrew Geissler82c905d2020-04-13 13:39:40 -0500173 self.configwatcher.bbseen = set()
174 self.configwatcher.bbwatchedfiles = set()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500175 self.confignotifier = pyinotify.Notifier(self.configwatcher, self.config_notifications)
Brad Bishopf058f492019-01-28 23:50:33 -0500176 bb.debug(1, "BBCooker pyinotify2 %s" % time.time())
177 sys.stdout.flush()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500178 self.watchmask = pyinotify.IN_CLOSE_WRITE | pyinotify.IN_CREATE | pyinotify.IN_DELETE | \
179 pyinotify.IN_DELETE_SELF | pyinotify.IN_MODIFY | pyinotify.IN_MOVE_SELF | \
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500180 pyinotify.IN_MOVED_FROM | pyinotify.IN_MOVED_TO
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500181 self.watcher = pyinotify.WatchManager()
Brad Bishopf058f492019-01-28 23:50:33 -0500182 bb.debug(1, "BBCooker pyinotify3 %s" % time.time())
183 sys.stdout.flush()
Andrew Geissler82c905d2020-04-13 13:39:40 -0500184 self.watcher.bbseen = set()
185 self.watcher.bbwatchedfiles = set()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500186 self.notifier = pyinotify.Notifier(self.watcher, self.notifications)
187
Brad Bishopf058f492019-01-28 23:50:33 -0500188 bb.debug(1, "BBCooker pyinotify complete %s" % time.time())
189 sys.stdout.flush()
190
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500191 # If being called by something like tinfoil, we need to clean cached data
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500192 # which may now be invalid
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500193 bb.parse.clear_cache()
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500194 bb.parse.BBHandler.cached_statements = {}
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500195
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500196 self.ui_cmdline = None
Brad Bishop08902b02019-08-20 09:16:51 -0400197 self.hashserv = None
Brad Bishopa34c0302019-09-23 22:34:48 -0400198 self.hashservaddr = None
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500199
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500200 self.inotify_modified_files = []
201
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500202 def _process_inotify_updates(server, cooker, abort):
203 cooker.process_inotify_updates()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500204 return 1.0
205
Andrew Geissler635e0e42020-08-21 15:58:33 -0500206 self.idleCallBackRegister(_process_inotify_updates, self)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500207
208 # TOSTOP must not be set or our children will hang when they output
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600209 try:
210 fd = sys.stdout.fileno()
211 if os.isatty(fd):
212 import termios
213 tcattr = termios.tcgetattr(fd)
214 if tcattr[3] & termios.TOSTOP:
215 buildlog.info("The terminal had the TOSTOP bit set, clearing...")
216 tcattr[3] = tcattr[3] & ~termios.TOSTOP
217 termios.tcsetattr(fd, termios.TCSANOW, tcattr)
218 except UnsupportedOperation:
219 pass
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500220
221 self.command = bb.command.Command(self)
222 self.state = state.initial
223
224 self.parser = None
225
226 signal.signal(signal.SIGTERM, self.sigterm_exception)
227 # Let SIGHUP exit as SIGTERM
228 signal.signal(signal.SIGHUP, self.sigterm_exception)
229
Brad Bishopf058f492019-01-28 23:50:33 -0500230 bb.debug(1, "BBCooker startup complete %s" % time.time())
231 sys.stdout.flush()
232
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500233 def init_configdata(self):
234 if not hasattr(self, "data"):
235 self.initConfigurationData()
236 bb.debug(1, "BBCooker parsed base configuration %s" % time.time())
237 sys.stdout.flush()
238 self.handlePRServ()
239
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500240 def process_inotify_updates(self):
241 for n in [self.confignotifier, self.notifier]:
242 if n.check_events(timeout=0):
243 # read notified events and enqeue them
244 n.read_events()
245 n.process_events()
246
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500247 def config_notifications(self, event):
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500248 if event.maskname == "IN_Q_OVERFLOW":
249 bb.warn("inotify event queue overflowed, invalidating caches.")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500250 self.parsecache_valid = False
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500251 self.baseconfig_valid = False
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500252 bb.parse.clear_cache()
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500253 return
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500254 if not event.pathname in self.configwatcher.bbwatchedfiles:
255 return
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500256 if not event.pathname in self.inotify_modified_files:
257 self.inotify_modified_files.append(event.pathname)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500258 self.baseconfig_valid = False
259
260 def notifications(self, event):
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500261 if event.maskname == "IN_Q_OVERFLOW":
262 bb.warn("inotify event queue overflowed, invalidating caches.")
263 self.parsecache_valid = False
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500264 bb.parse.clear_cache()
265 return
266 if event.pathname.endswith("bitbake-cookerdaemon.log") \
267 or event.pathname.endswith("bitbake.lock"):
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500268 return
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500269 if not event.pathname in self.inotify_modified_files:
270 self.inotify_modified_files.append(event.pathname)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500271 self.parsecache_valid = False
272
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500273 def add_filewatch(self, deps, watcher=None, dirs=False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500274 if not watcher:
275 watcher = self.watcher
276 for i in deps:
Andrew Geissler82c905d2020-04-13 13:39:40 -0500277 watcher.bbwatchedfiles.add(i[0])
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500278 if dirs:
279 f = i[0]
280 else:
281 f = os.path.dirname(i[0])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500282 if f in watcher.bbseen:
283 continue
Andrew Geissler82c905d2020-04-13 13:39:40 -0500284 watcher.bbseen.add(f)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500285 watchtarget = None
286 while True:
287 # We try and add watches for files that don't exist but if they did, would influence
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500288 # the parser. The parent directory of these files may not exist, in which case we need
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500289 # to watch any parent that does exist for changes.
290 try:
291 watcher.add_watch(f, self.watchmask, quiet=False)
292 if watchtarget:
Andrew Geissler82c905d2020-04-13 13:39:40 -0500293 watcher.bbwatchedfiles.add(watchtarget)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500294 break
295 except pyinotify.WatchManagerError as e:
296 if 'ENOENT' in str(e):
297 watchtarget = f
298 f = os.path.dirname(f)
299 if f in watcher.bbseen:
300 break
Andrew Geissler82c905d2020-04-13 13:39:40 -0500301 watcher.bbseen.add(f)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500302 continue
303 if 'ENOSPC' in str(e):
304 providerlog.error("No space left on device or exceeds fs.inotify.max_user_watches?")
305 providerlog.error("To check max_user_watches: sysctl -n fs.inotify.max_user_watches.")
306 providerlog.error("To modify max_user_watches: sysctl -n -w fs.inotify.max_user_watches=<value>.")
307 providerlog.error("Root privilege is required to modify max_user_watches.")
308 raise
309
310 def sigterm_exception(self, signum, stackframe):
311 if signum == signal.SIGTERM:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500312 bb.warn("Cooker received SIGTERM, shutting down...")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500313 elif signum == signal.SIGHUP:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500314 bb.warn("Cooker received SIGHUP, shutting down...")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500315 self.state = state.forceshutdown
316
317 def setFeatures(self, features):
318 # we only accept a new feature set if we're in state initial, so we can reset without problems
319 if not self.state in [state.initial, state.shutdown, state.forceshutdown, state.stopped, state.error]:
320 raise Exception("Illegal state for feature set change")
321 original_featureset = list(self.featureset)
322 for feature in features:
323 self.featureset.setFeature(feature)
324 bb.debug(1, "Features set %s (was %s)" % (original_featureset, list(self.featureset)))
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500325 if (original_featureset != list(self.featureset)) and self.state != state.error and hasattr(self, "data"):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500326 self.reset()
327
328 def initConfigurationData(self):
329
330 self.state = state.initial
331 self.caches_array = []
332
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500333 # Need to preserve BB_CONSOLELOG over resets
334 consolelog = None
335 if hasattr(self, "data"):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500336 consolelog = self.data.getVar("BB_CONSOLELOG")
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500337
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500338 if CookerFeatures.BASEDATASTORE_TRACKING in self.featureset:
339 self.enableDataTracking()
340
341 all_extra_cache_names = []
342 # We hardcode all known cache types in a single place, here.
343 if CookerFeatures.HOB_EXTRA_CACHES in self.featureset:
344 all_extra_cache_names.append("bb.cache_extra:HobRecipeInfo")
345
346 caches_name_array = ['bb.cache:CoreRecipeInfo'] + all_extra_cache_names
347
348 # At least CoreRecipeInfo will be loaded, so caches_array will never be empty!
349 # This is the entry point, no further check needed!
350 for var in caches_name_array:
351 try:
352 module_name, cache_name = var.split(':')
353 module = __import__(module_name, fromlist=(cache_name,))
354 self.caches_array.append(getattr(module, cache_name))
355 except ImportError as exc:
356 logger.critical("Unable to import extra RecipeInfo '%s' from '%s': %s" % (cache_name, module_name, exc))
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500357 raise bb.BBHandledException()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500358
359 self.databuilder = bb.cookerdata.CookerDataBuilder(self.configuration, False)
360 self.databuilder.parseBaseConfiguration()
361 self.data = self.databuilder.data
362 self.data_hash = self.databuilder.data_hash
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500363 self.extraconfigdata = {}
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500364
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500365 if consolelog:
366 self.data.setVar("BB_CONSOLELOG", consolelog)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500367
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500368 self.data.setVar('BB_CMDLINE', self.ui_cmdline)
369
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500370 if CookerFeatures.BASEDATASTORE_TRACKING in self.featureset:
371 self.disableDataTracking()
372
Brad Bishop15ae2502019-06-18 21:44:24 -0400373 for mc in self.databuilder.mcdata.values():
374 mc.renameVar("__depends", "__base_depends")
375 self.add_filewatch(mc.getVar("__base_depends", False), self.configwatcher)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500376
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500377 self.baseconfig_valid = True
378 self.parsecache_valid = False
379
380 def handlePRServ(self):
381 # Setup a PR Server based on the new configuration
382 try:
383 self.prhost = prserv.serv.auto_start(self.data)
384 except prserv.serv.PRServiceConfigError as e:
Andrew Geisslerd159c7f2021-09-02 21:05:58 -0500385 bb.fatal("Unable to start PR Server, exiting, check the bitbake-cookerdaemon.log")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500386
Brad Bishopa34c0302019-09-23 22:34:48 -0400387 if self.data.getVar("BB_HASHSERVE") == "auto":
388 # Create a new hash server bound to a unix domain socket
Brad Bishop08902b02019-08-20 09:16:51 -0400389 if not self.hashserv:
390 dbfile = (self.data.getVar("PERSISTENT_DIR") or self.data.getVar("CACHE")) + "/hashserv.db"
Brad Bishopa34c0302019-09-23 22:34:48 -0400391 self.hashservaddr = "unix://%s/hashserve.sock" % self.data.getVar("TOPDIR")
392 self.hashserv = hashserv.create_server(self.hashservaddr, dbfile, sync=False)
Patrick Williams213cb262021-08-07 19:21:33 -0500393 self.hashserv.serve_as_process()
Brad Bishopa34c0302019-09-23 22:34:48 -0400394 self.data.setVar("BB_HASHSERVE", self.hashservaddr)
395 self.databuilder.origdata.setVar("BB_HASHSERVE", self.hashservaddr)
396 self.databuilder.data.setVar("BB_HASHSERVE", self.hashservaddr)
Brad Bishop08902b02019-08-20 09:16:51 -0400397 for mc in self.databuilder.mcdata:
Brad Bishopa34c0302019-09-23 22:34:48 -0400398 self.databuilder.mcdata[mc].setVar("BB_HASHSERVE", self.hashservaddr)
Brad Bishop08902b02019-08-20 09:16:51 -0400399
400 bb.parse.init_parser(self.data)
401
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500402 def enableDataTracking(self):
403 self.configuration.tracking = True
404 if hasattr(self, "data"):
405 self.data.enableTracking()
406
407 def disableDataTracking(self):
408 self.configuration.tracking = False
409 if hasattr(self, "data"):
410 self.data.disableTracking()
411
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500412 def parseConfiguration(self):
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600413 self.updateCacheSync()
414
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500415 # Change nice level if we're asked to
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500416 nice = self.data.getVar("BB_NICE_LEVEL")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500417 if nice:
418 curnice = os.nice(0)
419 nice = int(nice) - curnice
420 buildlog.verbose("Renice to %s " % os.nice(nice))
421
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600422 if self.recipecaches:
423 del self.recipecaches
424 self.multiconfigs = self.databuilder.mcdata.keys()
425 self.recipecaches = {}
426 for mc in self.multiconfigs:
427 self.recipecaches[mc] = bb.cache.CacheData(self.caches_array)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500428
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500429 self.handleCollections(self.data.getVar("BBFILE_COLLECTIONS"))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500430
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500431 self.parsecache_valid = False
432
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500433 def updateConfigOpts(self, options, environment, cmdline):
434 self.ui_cmdline = cmdline
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500435 clean = True
436 for o in options:
437 if o in ['prefile', 'postfile']:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500438 # Only these options may require a reparse
439 try:
440 if getattr(self.configuration, o) == options[o]:
441 # Value is the same, no need to mark dirty
442 continue
443 except AttributeError:
444 pass
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600445 logger.debug("Marking as dirty due to '%s' option change to '%s'" % (o, options[o]))
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500446 print("Marking as dirty due to '%s' option change to '%s'" % (o, options[o]))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500447 clean = False
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500448 if hasattr(self.configuration, o):
449 setattr(self.configuration, o, options[o])
450
451 if self.configuration.writeeventlog:
452 if self.eventlog and self.eventlog[0] != self.configuration.writeeventlog:
453 bb.event.unregister_UIHhandler(self.eventlog[1])
454 if not self.eventlog or self.eventlog[0] != self.configuration.writeeventlog:
455 # we log all events to a file if so directed
456 # register the log file writer as UI Handler
457 writer = EventWriter(self, self.configuration.writeeventlog)
458 EventLogWriteHandler = namedtuple('EventLogWriteHandler', ['event'])
459 self.eventlog = (self.configuration.writeeventlog, bb.event.register_UIHhandler(EventLogWriteHandler(writer)))
460
461 bb.msg.loggerDefaultLogLevel = self.configuration.default_loglevel
462 bb.msg.loggerDefaultDomains = self.configuration.debug_domains
463
464 if hasattr(self, "data"):
465 origenv = bb.data.init()
466 for k in environment:
467 origenv.setVar(k, environment[k])
468 self.data.setVar("BB_ORIGENV", origenv)
469
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500470 for k in bb.utils.approved_variables():
471 if k in environment and k not in self.configuration.env:
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600472 logger.debug("Updating new environment variable %s to %s" % (k, environment[k]))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500473 self.configuration.env[k] = environment[k]
474 clean = False
475 if k in self.configuration.env and k not in environment:
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600476 logger.debug("Updating environment variable %s (deleted)" % (k))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500477 del self.configuration.env[k]
478 clean = False
479 if k not in self.configuration.env and k not in environment:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500480 continue
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500481 if environment[k] != self.configuration.env[k]:
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600482 logger.debug("Updating environment variable %s from %s to %s" % (k, self.configuration.env[k], environment[k]))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500483 self.configuration.env[k] = environment[k]
484 clean = False
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500485
486 # Now update all the variables not in the datastore to match
487 self.configuration.env = environment
488
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500489 if not clean:
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600490 logger.debug("Base environment change, triggering reparse")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500491 self.reset()
492
493 def runCommands(self, server, data, abort):
494 """
495 Run any queued asynchronous command
496 This is done by the idle handler so it runs in true context rather than
497 tied to any UI.
498 """
499
500 return self.command.runAsyncCommand()
501
502 def showVersions(self):
503
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500504 (latest_versions, preferred_versions, required) = self.findProviders()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500505
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500506 logger.plain("%-35s %25s %25s %25s", "Recipe Name", "Latest Version", "Preferred Version", "Required Version")
507 logger.plain("%-35s %25s %25s %25s\n", "===========", "==============", "=================", "================")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500508
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500509 for p in sorted(self.recipecaches[''].pkg_pn):
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500510 preferred = preferred_versions[p]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500511 latest = latest_versions[p]
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500512 requiredstr = ""
513 preferredstr = ""
514 if required[p]:
515 if preferred[0] is not None:
516 requiredstr = preferred[0][0] + ":" + preferred[0][1] + '-' + preferred[0][2]
517 else:
518 bb.fatal("REQUIRED_VERSION of package %s not available" % p)
519 else:
520 preferredstr = preferred[0][0] + ":" + preferred[0][1] + '-' + preferred[0][2]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500521
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500522 lateststr = latest[0][0] + ":" + latest[0][1] + "-" + latest[0][2]
523
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500524 if preferred == latest:
525 preferredstr = ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500526
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500527 logger.plain("%-35s %25s %25s %25s", p, lateststr, preferredstr, requiredstr)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500528
529 def showEnvironment(self, buildfile=None, pkgs_to_build=None):
530 """
531 Show the outer or per-recipe environment
532 """
533 fn = None
534 envdata = None
Brad Bishop15ae2502019-06-18 21:44:24 -0400535 mc = ''
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500536 if not pkgs_to_build:
537 pkgs_to_build = []
538
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500539 orig_tracking = self.configuration.tracking
540 if not orig_tracking:
541 self.enableDataTracking()
542 self.reset()
543
Brad Bishop15ae2502019-06-18 21:44:24 -0400544 def mc_base(p):
545 if p.startswith('mc:'):
546 s = p.split(':')
547 if len(s) == 2:
548 return s[1]
549 return None
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500550
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500551 if buildfile:
552 # Parse the configuration here. We need to do it explicitly here since
553 # this showEnvironment() code path doesn't use the cache
554 self.parseConfiguration()
555
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600556 fn, cls, mc = bb.cache.virtualfn2realfn(buildfile)
Andrew Geissler5a43b432020-06-13 10:46:56 -0500557 fn = self.matchFile(fn, mc)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600558 fn = bb.cache.realfn2virtual(fn, cls, mc)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500559 elif len(pkgs_to_build) == 1:
Brad Bishop15ae2502019-06-18 21:44:24 -0400560 mc = mc_base(pkgs_to_build[0])
561 if not mc:
562 ignore = self.data.getVar("ASSUME_PROVIDED") or ""
563 if pkgs_to_build[0] in set(ignore.split()):
564 bb.fatal("%s is in ASSUME_PROVIDED" % pkgs_to_build[0])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500565
Brad Bishop15ae2502019-06-18 21:44:24 -0400566 taskdata, runlist = self.buildTaskData(pkgs_to_build, None, self.configuration.abort, allowincomplete=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500567
Brad Bishop15ae2502019-06-18 21:44:24 -0400568 mc = runlist[0][0]
569 fn = runlist[0][3]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500570
571 if fn:
572 try:
Andrew Geissler5a43b432020-06-13 10:46:56 -0500573 bb_caches = bb.cache.MulticonfigCache(self.databuilder, self.data_hash, self.caches_array)
574 envdata = bb_caches[mc].loadDataFull(fn, self.collections[mc].get_file_appends(fn))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500575 except Exception as e:
576 parselog.exception("Unable to read %s", fn)
577 raise
Brad Bishop15ae2502019-06-18 21:44:24 -0400578 else:
579 if not mc in self.databuilder.mcdata:
580 bb.fatal('Not multiconfig named "%s" found' % mc)
581 envdata = self.databuilder.mcdata[mc]
582 data.expandKeys(envdata)
583 parse.ast.runAnonFuncs(envdata)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500584
585 # Display history
586 with closing(StringIO()) as env:
587 self.data.inchistory.emit(env)
588 logger.plain(env.getvalue())
589
590 # emit variables and shell functions
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500591 with closing(StringIO()) as env:
592 data.emit_env(env, envdata, True)
593 logger.plain(env.getvalue())
594
595 # emit the metadata which isnt valid shell
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500596 for e in sorted(envdata.keys()):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600597 if envdata.getVarFlag(e, 'func', False) and envdata.getVarFlag(e, 'python', False):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500598 logger.plain("\npython %s () {\n%s}\n", e, envdata.getVar(e, False))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500599
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500600 if not orig_tracking:
601 self.disableDataTracking()
602 self.reset()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500603
604 def buildTaskData(self, pkgs_to_build, task, abort, allowincomplete=False):
605 """
606 Prepare a runqueue and taskdata object for iteration over pkgs_to_build
607 """
608 bb.event.fire(bb.event.TreeDataPreparationStarted(), self.data)
609
610 # A task of None means use the default task
611 if task is None:
612 task = self.configuration.cmd
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500613 if not task.startswith("do_"):
614 task = "do_%s" % task
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500615
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500616 targetlist = self.checkPackages(pkgs_to_build, task)
617 fulltargetlist = []
618 defaulttask_implicit = ''
619 defaulttask_explicit = False
620 wildcard = False
621
622 # Wild card expansion:
Brad Bishop15ae2502019-06-18 21:44:24 -0400623 # Replace string such as "mc:*:bash"
624 # into "mc:A:bash mc:B:bash bash"
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500625 for k in targetlist:
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600626 if k.startswith("mc:") and k.count(':') >= 2:
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500627 if wildcard:
628 bb.fatal('multiconfig conflict')
629 if k.split(":")[1] == "*":
630 wildcard = True
631 for mc in self.multiconfigs:
632 if mc:
633 fulltargetlist.append(k.replace('*', mc))
634 # implicit default task
635 else:
636 defaulttask_implicit = k.split(":")[2]
637 else:
638 fulltargetlist.append(k)
639 else:
640 defaulttask_explicit = True
641 fulltargetlist.append(k)
642
643 if not defaulttask_explicit and defaulttask_implicit != '':
644 fulltargetlist.append(defaulttask_implicit)
645
646 bb.debug(1,"Target list: %s" % (str(fulltargetlist)))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600647 taskdata = {}
648 localdata = {}
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500649
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600650 for mc in self.multiconfigs:
651 taskdata[mc] = bb.taskdata.TaskData(abort, skiplist=self.skiplist, allowincomplete=allowincomplete)
652 localdata[mc] = data.createCopy(self.databuilder.mcdata[mc])
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600653 bb.data.expandKeys(localdata[mc])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500654
655 current = 0
656 runlist = []
657 for k in fulltargetlist:
Andrew Geisslerb7d28612020-07-24 16:15:54 -0500658 origk = k
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600659 mc = ""
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600660 if k.startswith("mc:") and k.count(':') >= 2:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600661 mc = k.split(":")[1]
662 k = ":".join(k.split(":")[2:])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500663 ktask = task
664 if ":do_" in k:
665 k2 = k.split(":do_")
666 k = k2[0]
667 ktask = k2[1]
Andrew Geisslerb7d28612020-07-24 16:15:54 -0500668
669 if mc not in self.multiconfigs:
670 bb.fatal("Multiconfig dependency %s depends on nonexistent multiconfig configuration named %s" % (origk, mc))
671
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600672 taskdata[mc].add_provider(localdata[mc], self.recipecaches[mc], k)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500673 current += 1
674 if not ktask.startswith("do_"):
675 ktask = "do_%s" % ktask
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600676 if k not in taskdata[mc].build_targets or not taskdata[mc].build_targets[k]:
677 # e.g. in ASSUME_PROVIDED
678 continue
679 fn = taskdata[mc].build_targets[k][0]
680 runlist.append([mc, k, ktask, fn])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500681 bb.event.fire(bb.event.TreeDataPreparationProgress(current, len(fulltargetlist)), self.data)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600682
Brad Bishop15ae2502019-06-18 21:44:24 -0400683 havemc = False
684 for mc in self.multiconfigs:
685 if taskdata[mc].get_mcdepends():
686 havemc = True
Brad Bishopf058f492019-01-28 23:50:33 -0500687
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800688 # No need to do check providers if there are no mcdeps or not an mc build
Brad Bishop15ae2502019-06-18 21:44:24 -0400689 if havemc or len(self.multiconfigs) > 1:
Andrew Geissler99467da2019-02-25 18:54:23 -0600690 seen = set()
691 new = True
692 # Make sure we can provide the multiconfig dependency
693 while new:
694 mcdeps = set()
695 # Add unresolved first, so we can get multiconfig indirect dependencies on time
696 for mc in self.multiconfigs:
697 taskdata[mc].add_unresolved(localdata[mc], self.recipecaches[mc])
698 mcdeps |= set(taskdata[mc].get_mcdepends())
699 new = False
700 for mc in self.multiconfigs:
701 for k in mcdeps:
702 if k in seen:
703 continue
704 l = k.split(':')
705 depmc = l[2]
706 if depmc not in self.multiconfigs:
Andrew Geisslerb7d28612020-07-24 16:15:54 -0500707 bb.fatal("Multiconfig dependency %s depends on nonexistent multiconfig configuration named configuration %s" % (k,depmc))
Andrew Geissler99467da2019-02-25 18:54:23 -0600708 else:
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600709 logger.debug("Adding providers for multiconfig dependency %s" % l[3])
Andrew Geissler99467da2019-02-25 18:54:23 -0600710 taskdata[depmc].add_provider(localdata[depmc], self.recipecaches[depmc], l[3])
711 seen.add(k)
712 new = True
Brad Bishopf058f492019-01-28 23:50:33 -0500713
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600714 for mc in self.multiconfigs:
715 taskdata[mc].add_unresolved(localdata[mc], self.recipecaches[mc])
716
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500717 bb.event.fire(bb.event.TreeDataPreparationCompleted(len(fulltargetlist)), self.data)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600718 return taskdata, runlist
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500719
720 def prepareTreeData(self, pkgs_to_build, task):
721 """
722 Prepare a runqueue and taskdata object for iteration over pkgs_to_build
723 """
724
725 # We set abort to False here to prevent unbuildable targets raising
726 # an exception when we're just generating data
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600727 taskdata, runlist = self.buildTaskData(pkgs_to_build, task, False, allowincomplete=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500728
729 return runlist, taskdata
730
731 ######## WARNING : this function requires cache_extra to be enabled ########
732
733 def generateTaskDepTreeData(self, pkgs_to_build, task):
734 """
735 Create a dependency graph of pkgs_to_build including reverse dependency
736 information.
737 """
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500738 if not task.startswith("do_"):
739 task = "do_%s" % task
740
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500741 runlist, taskdata = self.prepareTreeData(pkgs_to_build, task)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600742 rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500743 rq.rqdata.prepare()
744 return self.buildDependTree(rq, taskdata)
745
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600746 @staticmethod
747 def add_mc_prefix(mc, pn):
748 if mc:
Brad Bishop15ae2502019-06-18 21:44:24 -0400749 return "mc:%s:%s" % (mc, pn)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600750 return pn
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500751
752 def buildDependTree(self, rq, taskdata):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600753 seen_fns = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500754 depend_tree = {}
755 depend_tree["depends"] = {}
756 depend_tree["tdepends"] = {}
757 depend_tree["pn"] = {}
758 depend_tree["rdepends-pn"] = {}
759 depend_tree["packages"] = {}
760 depend_tree["rdepends-pkg"] = {}
761 depend_tree["rrecs-pkg"] = {}
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500762 depend_tree['providermap'] = {}
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600763 depend_tree["layer-priorities"] = self.bbfile_config_priorities
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500764
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600765 for mc in taskdata:
766 for name, fn in list(taskdata[mc].get_providermap().items()):
767 pn = self.recipecaches[mc].pkg_fn[fn]
768 pn = self.add_mc_prefix(mc, pn)
769 if name != pn:
770 version = "%s:%s-%s" % self.recipecaches[mc].pkg_pepvpr[fn]
771 depend_tree['providermap'][name] = (pn, version)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500772
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600773 for tid in rq.rqdata.runtaskentries:
774 (mc, fn, taskname, taskfn) = bb.runqueue.split_tid_mcfn(tid)
775 pn = self.recipecaches[mc].pkg_fn[taskfn]
776 pn = self.add_mc_prefix(mc, pn)
777 version = "%s:%s-%s" % self.recipecaches[mc].pkg_pepvpr[taskfn]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500778 if pn not in depend_tree["pn"]:
779 depend_tree["pn"][pn] = {}
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600780 depend_tree["pn"][pn]["filename"] = taskfn
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500781 depend_tree["pn"][pn]["version"] = version
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600782 depend_tree["pn"][pn]["inherits"] = self.recipecaches[mc].inherits.get(taskfn, None)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500783
784 # if we have extra caches, list all attributes they bring in
785 extra_info = []
786 for cache_class in self.caches_array:
787 if type(cache_class) is type and issubclass(cache_class, bb.cache.RecipeInfoCommon) and hasattr(cache_class, 'cachefields'):
788 cachefields = getattr(cache_class, 'cachefields', [])
789 extra_info = extra_info + cachefields
790
791 # for all attributes stored, add them to the dependency tree
792 for ei in extra_info:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600793 depend_tree["pn"][pn][ei] = vars(self.recipecaches[mc])[ei][taskfn]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500794
795
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500796 dotname = "%s.%s" % (pn, bb.runqueue.taskname_from_tid(tid))
797 if not dotname in depend_tree["tdepends"]:
798 depend_tree["tdepends"][dotname] = []
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600799 for dep in rq.rqdata.runtaskentries[tid].depends:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800800 (depmc, depfn, _, deptaskfn) = bb.runqueue.split_tid_mcfn(dep)
801 deppn = self.recipecaches[depmc].pkg_fn[deptaskfn]
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600802 depend_tree["tdepends"][dotname].append("%s.%s" % (deppn, bb.runqueue.taskname_from_tid(dep)))
803 if taskfn not in seen_fns:
804 seen_fns.append(taskfn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500805 packages = []
806
807 depend_tree["depends"][pn] = []
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600808 for dep in taskdata[mc].depids[taskfn]:
809 depend_tree["depends"][pn].append(dep)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500810
811 depend_tree["rdepends-pn"][pn] = []
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600812 for rdep in taskdata[mc].rdepids[taskfn]:
813 depend_tree["rdepends-pn"][pn].append(rdep)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500814
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600815 rdepends = self.recipecaches[mc].rundeps[taskfn]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500816 for package in rdepends:
817 depend_tree["rdepends-pkg"][package] = []
818 for rdepend in rdepends[package]:
819 depend_tree["rdepends-pkg"][package].append(rdepend)
820 packages.append(package)
821
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600822 rrecs = self.recipecaches[mc].runrecs[taskfn]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500823 for package in rrecs:
824 depend_tree["rrecs-pkg"][package] = []
825 for rdepend in rrecs[package]:
826 depend_tree["rrecs-pkg"][package].append(rdepend)
827 if not package in packages:
828 packages.append(package)
829
830 for package in packages:
831 if package not in depend_tree["packages"]:
832 depend_tree["packages"][package] = {}
833 depend_tree["packages"][package]["pn"] = pn
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600834 depend_tree["packages"][package]["filename"] = taskfn
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500835 depend_tree["packages"][package]["version"] = version
836
837 return depend_tree
838
839 ######## WARNING : this function requires cache_extra to be enabled ########
840 def generatePkgDepTreeData(self, pkgs_to_build, task):
841 """
842 Create a dependency tree of pkgs_to_build, returning the data.
843 """
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500844 if not task.startswith("do_"):
845 task = "do_%s" % task
846
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500847 _, taskdata = self.prepareTreeData(pkgs_to_build, task)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500848
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600849 seen_fns = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500850 depend_tree = {}
851 depend_tree["depends"] = {}
852 depend_tree["pn"] = {}
853 depend_tree["rdepends-pn"] = {}
854 depend_tree["rdepends-pkg"] = {}
855 depend_tree["rrecs-pkg"] = {}
856
857 # if we have extra caches, list all attributes they bring in
858 extra_info = []
859 for cache_class in self.caches_array:
860 if type(cache_class) is type and issubclass(cache_class, bb.cache.RecipeInfoCommon) and hasattr(cache_class, 'cachefields'):
861 cachefields = getattr(cache_class, 'cachefields', [])
862 extra_info = extra_info + cachefields
863
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600864 tids = []
865 for mc in taskdata:
866 for tid in taskdata[mc].taskentries:
867 tids.append(tid)
868
869 for tid in tids:
870 (mc, fn, taskname, taskfn) = bb.runqueue.split_tid_mcfn(tid)
871
872 pn = self.recipecaches[mc].pkg_fn[taskfn]
873 pn = self.add_mc_prefix(mc, pn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500874
875 if pn not in depend_tree["pn"]:
876 depend_tree["pn"][pn] = {}
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600877 depend_tree["pn"][pn]["filename"] = taskfn
878 version = "%s:%s-%s" % self.recipecaches[mc].pkg_pepvpr[taskfn]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500879 depend_tree["pn"][pn]["version"] = version
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600880 rdepends = self.recipecaches[mc].rundeps[taskfn]
881 rrecs = self.recipecaches[mc].runrecs[taskfn]
882 depend_tree["pn"][pn]["inherits"] = self.recipecaches[mc].inherits.get(taskfn, None)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500883
884 # for all extra attributes stored, add them to the dependency tree
885 for ei in extra_info:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600886 depend_tree["pn"][pn][ei] = vars(self.recipecaches[mc])[ei][taskfn]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500887
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600888 if taskfn not in seen_fns:
889 seen_fns.append(taskfn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500890
891 depend_tree["depends"][pn] = []
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500892 for dep in taskdata[mc].depids[taskfn]:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500893 pn_provider = ""
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600894 if dep in taskdata[mc].build_targets and taskdata[mc].build_targets[dep]:
895 fn_provider = taskdata[mc].build_targets[dep][0]
896 pn_provider = self.recipecaches[mc].pkg_fn[fn_provider]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500897 else:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500898 pn_provider = dep
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600899 pn_provider = self.add_mc_prefix(mc, pn_provider)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500900 depend_tree["depends"][pn].append(pn_provider)
901
902 depend_tree["rdepends-pn"][pn] = []
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600903 for rdep in taskdata[mc].rdepids[taskfn]:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500904 pn_rprovider = ""
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600905 if rdep in taskdata[mc].run_targets and taskdata[mc].run_targets[rdep]:
906 fn_rprovider = taskdata[mc].run_targets[rdep][0]
907 pn_rprovider = self.recipecaches[mc].pkg_fn[fn_rprovider]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500908 else:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600909 pn_rprovider = rdep
910 pn_rprovider = self.add_mc_prefix(mc, pn_rprovider)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500911 depend_tree["rdepends-pn"][pn].append(pn_rprovider)
912
913 depend_tree["rdepends-pkg"].update(rdepends)
914 depend_tree["rrecs-pkg"].update(rrecs)
915
916 return depend_tree
917
918 def generateDepTreeEvent(self, pkgs_to_build, task):
919 """
920 Create a task dependency graph of pkgs_to_build.
921 Generate an event with the result
922 """
923 depgraph = self.generateTaskDepTreeData(pkgs_to_build, task)
924 bb.event.fire(bb.event.DepTreeGenerated(depgraph), self.data)
925
926 def generateDotGraphFiles(self, pkgs_to_build, task):
927 """
928 Create a task dependency graph of pkgs_to_build.
929 Save the result to a set of .dot files.
930 """
931
932 depgraph = self.generateTaskDepTreeData(pkgs_to_build, task)
933
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500934 with open('pn-buildlist', 'w') as f:
935 for pn in depgraph["pn"]:
936 f.write(pn + "\n")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500937 logger.info("PN build list saved to 'pn-buildlist'")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500938
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500939 # Remove old format output files to ensure no confusion with stale data
940 try:
941 os.unlink('pn-depends.dot')
942 except FileNotFoundError:
943 pass
944 try:
945 os.unlink('package-depends.dot')
946 except FileNotFoundError:
947 pass
Brad Bishop79641f22019-09-10 07:20:22 -0400948 try:
949 os.unlink('recipe-depends.dot')
950 except FileNotFoundError:
951 pass
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500952
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500953 with open('task-depends.dot', 'w') as f:
954 f.write("digraph depends {\n")
Brad Bishop316dfdd2018-06-25 12:45:53 -0400955 for task in sorted(depgraph["tdepends"]):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500956 (pn, taskname) = task.rsplit(".", 1)
957 fn = depgraph["pn"][pn]["filename"]
958 version = depgraph["pn"][pn]["version"]
959 f.write('"%s.%s" [label="%s %s\\n%s\\n%s"]\n' % (pn, taskname, pn, taskname, version, fn))
Brad Bishop316dfdd2018-06-25 12:45:53 -0400960 for dep in sorted(depgraph["tdepends"][task]):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500961 f.write('"%s" -> "%s"\n' % (task, dep))
962 f.write("}\n")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500963 logger.info("Task dependencies saved to 'task-depends.dot'")
964
965 def show_appends_with_no_recipes(self):
Andrew Geissler5a43b432020-06-13 10:46:56 -0500966 appends_without_recipes = {}
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500967 # Determine which bbappends haven't been applied
Andrew Geissler5a43b432020-06-13 10:46:56 -0500968 for mc in self.multiconfigs:
969 # First get list of recipes, including skipped
970 recipefns = list(self.recipecaches[mc].pkg_fn.keys())
971 recipefns.extend(self.skiplist.keys())
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500972
Andrew Geissler5a43b432020-06-13 10:46:56 -0500973 # Work out list of bbappends that have been applied
974 applied_appends = []
975 for fn in recipefns:
976 applied_appends.extend(self.collections[mc].get_file_appends(fn))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500977
Andrew Geissler5a43b432020-06-13 10:46:56 -0500978 appends_without_recipes[mc] = []
979 for _, appendfn in self.collections[mc].bbappends:
980 if not appendfn in applied_appends:
981 appends_without_recipes[mc].append(appendfn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500982
Andrew Geissler5a43b432020-06-13 10:46:56 -0500983 msgs = []
984 for mc in sorted(appends_without_recipes.keys()):
985 if appends_without_recipes[mc]:
986 msgs.append('No recipes in %s available for:\n %s' % (mc if mc else 'default',
987 '\n '.join(appends_without_recipes[mc])))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500988
Andrew Geissler5a43b432020-06-13 10:46:56 -0500989 if msgs:
990 msg = "\n".join(msgs)
991 warn_only = self.databuilder.mcdata[mc].getVar("BB_DANGLINGAPPENDS_WARNONLY", \
992 False) or "no"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500993 if warn_only.lower() in ("1", "yes", "true"):
994 bb.warn(msg)
995 else:
996 bb.fatal(msg)
997
998 def handlePrefProviders(self):
999
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001000 for mc in self.multiconfigs:
1001 localdata = data.createCopy(self.databuilder.mcdata[mc])
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001002 bb.data.expandKeys(localdata)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001003
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001004 # Handle PREFERRED_PROVIDERS
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001005 for p in (localdata.getVar('PREFERRED_PROVIDERS') or "").split():
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001006 try:
1007 (providee, provider) = p.split(':')
1008 except:
1009 providerlog.critical("Malformed option in PREFERRED_PROVIDERS variable: %s" % p)
1010 continue
1011 if providee in self.recipecaches[mc].preferred and self.recipecaches[mc].preferred[providee] != provider:
1012 providerlog.error("conflicting preferences for %s: both %s and %s specified", providee, provider, self.recipecaches[mc].preferred[providee])
1013 self.recipecaches[mc].preferred[providee] = provider
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001014
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001015 def findConfigFilePath(self, configfile):
1016 """
1017 Find the location on disk of configfile and if it exists and was parsed by BitBake
1018 emit the ConfigFilePathFound event with the path to the file.
1019 """
1020 path = bb.cookerdata.findConfigFile(configfile, self.data)
1021 if not path:
1022 return
1023
1024 # Generate a list of parsed configuration files by searching the files
1025 # listed in the __depends and __base_depends variables with a .conf suffix.
1026 conffiles = []
1027 dep_files = self.data.getVar('__base_depends', False) or []
1028 dep_files = dep_files + (self.data.getVar('__depends', False) or [])
1029
1030 for f in dep_files:
1031 if f[0].endswith(".conf"):
1032 conffiles.append(f[0])
1033
1034 _, conf, conffile = path.rpartition("conf/")
1035 match = os.path.join(conf, conffile)
1036 # Try and find matches for conf/conffilename.conf as we don't always
1037 # have the full path to the file.
1038 for cfg in conffiles:
1039 if cfg.endswith(match):
1040 bb.event.fire(bb.event.ConfigFilePathFound(path),
1041 self.data)
1042 break
1043
1044 def findFilesMatchingInDir(self, filepattern, directory):
1045 """
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001046 Searches for files containing the substring 'filepattern' which are children of
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001047 'directory' in each BBPATH. i.e. to find all rootfs package classes available
1048 to BitBake one could call findFilesMatchingInDir(self, 'rootfs_', 'classes')
1049 or to find all machine configuration files one could call:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001050 findFilesMatchingInDir(self, '.conf', 'conf/machine')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001051 """
1052
1053 matches = []
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001054 bbpaths = self.data.getVar('BBPATH').split(':')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001055 for path in bbpaths:
1056 dirpath = os.path.join(path, directory)
1057 if os.path.exists(dirpath):
1058 for root, dirs, files in os.walk(dirpath):
1059 for f in files:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001060 if filepattern in f:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001061 matches.append(f)
1062
1063 if matches:
1064 bb.event.fire(bb.event.FilesMatchingFound(filepattern, matches), self.data)
1065
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001066 def findProviders(self, mc=''):
Andrew Geissler82c905d2020-04-13 13:39:40 -05001067 return bb.providers.findProviders(self.databuilder.mcdata[mc], self.recipecaches[mc], self.recipecaches[mc].pkg_pn)
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001068
1069 def findBestProvider(self, pn, mc=''):
1070 if pn in self.recipecaches[mc].providers:
1071 filenames = self.recipecaches[mc].providers[pn]
Andrew Geissler82c905d2020-04-13 13:39:40 -05001072 eligible, foundUnique = bb.providers.filterProviders(filenames, pn, self.databuilder.mcdata[mc], self.recipecaches[mc])
Andrew Geissler95ac1b82021-03-31 14:34:31 -05001073 if eligible is not None:
1074 filename = eligible[0]
1075 else:
1076 filename = None
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001077 return None, None, None, filename
1078 elif pn in self.recipecaches[mc].pkg_pn:
Andrew Geissler95ac1b82021-03-31 14:34:31 -05001079 (latest, latest_f, preferred_ver, preferred_file, required) = bb.providers.findBestProvider(pn, self.databuilder.mcdata[mc], self.recipecaches[mc], self.recipecaches[mc].pkg_pn)
1080 if required and preferred_file is None:
1081 return None, None, None, None
1082 return (latest, latest_f, preferred_ver, preferred_file)
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001083 else:
1084 return None, None, None, None
1085
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001086 def findConfigFiles(self, varname):
1087 """
1088 Find config files which are appropriate values for varname.
1089 i.e. MACHINE, DISTRO
1090 """
1091 possible = []
1092 var = varname.lower()
1093
1094 data = self.data
1095 # iterate configs
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001096 bbpaths = data.getVar('BBPATH').split(':')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001097 for path in bbpaths:
1098 confpath = os.path.join(path, "conf", var)
1099 if os.path.exists(confpath):
1100 for root, dirs, files in os.walk(confpath):
1101 # get all child files, these are appropriate values
1102 for f in files:
1103 val, sep, end = f.rpartition('.')
1104 if end == 'conf':
1105 possible.append(val)
1106
1107 if possible:
1108 bb.event.fire(bb.event.ConfigFilesFound(var, possible), self.data)
1109
1110 def findInheritsClass(self, klass):
1111 """
1112 Find all recipes which inherit the specified class
1113 """
1114 pkg_list = []
1115
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001116 for pfn in self.recipecaches[''].pkg_fn:
1117 inherits = self.recipecaches[''].inherits.get(pfn, None)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001118 if inherits and klass in inherits:
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001119 pkg_list.append(self.recipecaches[''].pkg_fn[pfn])
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001120
1121 return pkg_list
1122
1123 def generateTargetsTree(self, klass=None, pkgs=None):
1124 """
1125 Generate a dependency tree of buildable targets
1126 Generate an event with the result
1127 """
1128 # if the caller hasn't specified a pkgs list default to universe
1129 if not pkgs:
1130 pkgs = ['universe']
1131 # if inherited_class passed ensure all recipes which inherit the
1132 # specified class are included in pkgs
1133 if klass:
1134 extra_pkgs = self.findInheritsClass(klass)
1135 pkgs = pkgs + extra_pkgs
1136
1137 # generate a dependency tree for all our packages
1138 tree = self.generatePkgDepTreeData(pkgs, 'build')
1139 bb.event.fire(bb.event.TargetsTreeGenerated(tree), self.data)
1140
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001141 def interactiveMode( self ):
1142 """Drop off into a shell"""
1143 try:
1144 from bb import shell
1145 except ImportError:
1146 parselog.exception("Interactive mode not available")
Andrew Geisslerc9f78652020-09-18 14:11:35 -05001147 raise bb.BBHandledException()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001148 else:
1149 shell.start( self )
1150
1151
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001152 def handleCollections(self, collections):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001153 """Handle collections"""
1154 errors = False
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001155 self.bbfile_config_priorities = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001156 if collections:
1157 collection_priorities = {}
1158 collection_depends = {}
1159 collection_list = collections.split()
1160 min_prio = 0
1161 for c in collection_list:
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001162 bb.debug(1,'Processing %s in collection list' % (c))
1163
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001164 # Get collection priority if defined explicitly
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001165 priority = self.data.getVar("BBFILE_PRIORITY_%s" % c)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001166 if priority:
1167 try:
1168 prio = int(priority)
1169 except ValueError:
1170 parselog.error("invalid value for BBFILE_PRIORITY_%s: \"%s\"", c, priority)
1171 errors = True
1172 if min_prio == 0 or prio < min_prio:
1173 min_prio = prio
1174 collection_priorities[c] = prio
1175 else:
1176 collection_priorities[c] = None
1177
1178 # Check dependencies and store information for priority calculation
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001179 deps = self.data.getVar("LAYERDEPENDS_%s" % c)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001180 if deps:
1181 try:
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001182 depDict = bb.utils.explode_dep_versions2(deps)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001183 except bb.utils.VersionStringException as vse:
1184 bb.fatal('Error parsing LAYERDEPENDS_%s: %s' % (c, str(vse)))
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001185 for dep, oplist in list(depDict.items()):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001186 if dep in collection_list:
1187 for opstr in oplist:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001188 layerver = self.data.getVar("LAYERVERSION_%s" % dep)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001189 (op, depver) = opstr.split()
1190 if layerver:
1191 try:
1192 res = bb.utils.vercmp_string_op(layerver, depver, op)
1193 except bb.utils.VersionStringException as vse:
1194 bb.fatal('Error parsing LAYERDEPENDS_%s: %s' % (c, str(vse)))
1195 if not res:
1196 parselog.error("Layer '%s' depends on version %s of layer '%s', but version %s is currently enabled in your configuration. Check that you are using the correct matching versions/branches of these two layers.", c, opstr, dep, layerver)
1197 errors = True
1198 else:
1199 parselog.error("Layer '%s' depends on version %s of layer '%s', which exists in your configuration but does not specify a version. Check that you are using the correct matching versions/branches of these two layers.", c, opstr, dep)
1200 errors = True
1201 else:
1202 parselog.error("Layer '%s' depends on layer '%s', but this layer is not enabled in your configuration", c, dep)
1203 errors = True
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001204 collection_depends[c] = list(depDict.keys())
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001205 else:
1206 collection_depends[c] = []
1207
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001208 # Check recommends and store information for priority calculation
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001209 recs = self.data.getVar("LAYERRECOMMENDS_%s" % c)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001210 if recs:
1211 try:
1212 recDict = bb.utils.explode_dep_versions2(recs)
1213 except bb.utils.VersionStringException as vse:
1214 bb.fatal('Error parsing LAYERRECOMMENDS_%s: %s' % (c, str(vse)))
1215 for rec, oplist in list(recDict.items()):
1216 if rec in collection_list:
1217 if oplist:
1218 opstr = oplist[0]
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001219 layerver = self.data.getVar("LAYERVERSION_%s" % rec)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001220 if layerver:
1221 (op, recver) = opstr.split()
1222 try:
1223 res = bb.utils.vercmp_string_op(layerver, recver, op)
1224 except bb.utils.VersionStringException as vse:
1225 bb.fatal('Error parsing LAYERRECOMMENDS_%s: %s' % (c, str(vse)))
1226 if not res:
1227 parselog.debug(3,"Layer '%s' recommends version %s of layer '%s', but version %s is currently enabled in your configuration. Check that you are using the correct matching versions/branches of these two layers.", c, opstr, rec, layerver)
1228 continue
1229 else:
1230 parselog.debug(3,"Layer '%s' recommends version %s of layer '%s', which exists in your configuration but does not specify a version. Check that you are using the correct matching versions/branches of these two layers.", c, opstr, rec)
1231 continue
1232 parselog.debug(3,"Layer '%s' recommends layer '%s', so we are adding it", c, rec)
1233 collection_depends[c].append(rec)
1234 else:
1235 parselog.debug(3,"Layer '%s' recommends layer '%s', but this layer is not enabled in your configuration", c, rec)
1236
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001237 # Recursively work out collection priorities based on dependencies
1238 def calc_layer_priority(collection):
1239 if not collection_priorities[collection]:
1240 max_depprio = min_prio
1241 for dep in collection_depends[collection]:
1242 calc_layer_priority(dep)
1243 depprio = collection_priorities[dep]
1244 if depprio > max_depprio:
1245 max_depprio = depprio
1246 max_depprio += 1
1247 parselog.debug(1, "Calculated priority of layer %s as %d", collection, max_depprio)
1248 collection_priorities[collection] = max_depprio
1249
1250 # Calculate all layer priorities using calc_layer_priority and store in bbfile_config_priorities
1251 for c in collection_list:
1252 calc_layer_priority(c)
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001253 regex = self.data.getVar("BBFILE_PATTERN_%s" % c)
Andrew Geissler82c905d2020-04-13 13:39:40 -05001254 if regex is None:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001255 parselog.error("BBFILE_PATTERN_%s not defined" % c)
1256 errors = True
1257 continue
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001258 elif regex == "":
1259 parselog.debug(1, "BBFILE_PATTERN_%s is empty" % c)
Brad Bishop19323692019-04-05 15:28:33 -04001260 cre = re.compile('^NULL$')
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001261 errors = False
1262 else:
1263 try:
1264 cre = re.compile(regex)
1265 except re.error:
1266 parselog.error("BBFILE_PATTERN_%s \"%s\" is not a valid regular expression", c, regex)
1267 errors = True
1268 continue
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001269 self.bbfile_config_priorities.append((c, regex, cre, collection_priorities[c]))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001270 if errors:
1271 # We've already printed the actual error(s)
1272 raise CollectionError("Errors during parsing layer configuration")
1273
1274 def buildSetVars(self):
1275 """
1276 Setup any variables needed before starting a build
1277 """
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001278 t = time.gmtime()
1279 for mc in self.databuilder.mcdata:
1280 ds = self.databuilder.mcdata[mc]
1281 if not ds.getVar("BUILDNAME", False):
1282 ds.setVar("BUILDNAME", "${DATE}${TIME}")
1283 ds.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S', t))
1284 ds.setVar("DATE", time.strftime('%Y%m%d', t))
1285 ds.setVar("TIME", time.strftime('%H%M%S', t))
1286
1287 def reset_mtime_caches(self):
1288 """
1289 Reset mtime caches - this is particularly important when memory resident as something
1290 which is cached is not unlikely to have changed since the last invocation (e.g. a
1291 file associated with a recipe might have been modified by the user).
1292 """
1293 build.reset_cache()
1294 bb.fetch._checksum_cache.mtime_cache.clear()
1295 siggen_cache = getattr(bb.parse.siggen, 'checksum_cache', None)
1296 if siggen_cache:
1297 bb.parse.siggen.checksum_cache.mtime_cache.clear()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001298
Andrew Geissler5a43b432020-06-13 10:46:56 -05001299 def matchFiles(self, bf, mc=''):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001300 """
1301 Find the .bb files which match the expression in 'buildfile'.
1302 """
1303 if bf.startswith("/") or bf.startswith("../"):
1304 bf = os.path.abspath(bf)
1305
Andrew Geissler5a43b432020-06-13 10:46:56 -05001306 self.collections = {mc: CookerCollectFiles(self.bbfile_config_priorities, mc)}
1307 filelist, masked, searchdirs = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc])
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001308 try:
1309 os.stat(bf)
1310 bf = os.path.abspath(bf)
1311 return [bf]
1312 except OSError:
1313 regexp = re.compile(bf)
1314 matches = []
1315 for f in filelist:
1316 if regexp.search(f) and os.path.isfile(f):
1317 matches.append(f)
1318 return matches
1319
Andrew Geissler5a43b432020-06-13 10:46:56 -05001320 def matchFile(self, buildfile, mc=''):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001321 """
1322 Find the .bb file which matches the expression in 'buildfile'.
1323 Raise an error if multiple files
1324 """
Andrew Geissler5a43b432020-06-13 10:46:56 -05001325 matches = self.matchFiles(buildfile, mc)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001326 if len(matches) != 1:
1327 if matches:
1328 msg = "Unable to match '%s' to a specific recipe file - %s matches found:" % (buildfile, len(matches))
1329 if matches:
1330 for f in matches:
1331 msg += "\n %s" % f
1332 parselog.error(msg)
1333 else:
1334 parselog.error("Unable to find any recipe file matching '%s'" % buildfile)
1335 raise NoSpecificMatch
1336 return matches[0]
1337
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001338 def buildFile(self, buildfile, task):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001339 """
1340 Build the file matching regexp buildfile
1341 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001342 bb.event.fire(bb.event.BuildInit(), self.data)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001343
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001344 # Too many people use -b because they think it's how you normally
1345 # specify a target to be built, so show a warning
1346 bb.warn("Buildfile specified, dependencies will not be handled. If this is not what you want, do not use -b / --buildfile.")
1347
1348 self.buildFileInternal(buildfile, task)
1349
1350 def buildFileInternal(self, buildfile, task, fireevents=True, quietlog=False):
1351 """
1352 Build the file matching regexp buildfile
1353 """
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001354
1355 # Parse the configuration here. We need to do it explicitly here since
1356 # buildFile() doesn't use the cache
1357 self.parseConfiguration()
1358
1359 # If we are told to do the None task then query the default task
Andrew Geissler82c905d2020-04-13 13:39:40 -05001360 if task is None:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001361 task = self.configuration.cmd
Brad Bishop37a0e4d2017-12-04 01:01:44 -05001362 if not task.startswith("do_"):
1363 task = "do_%s" % task
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001364
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001365 fn, cls, mc = bb.cache.virtualfn2realfn(buildfile)
Andrew Geissler5a43b432020-06-13 10:46:56 -05001366 fn = self.matchFile(fn, mc)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001367
1368 self.buildSetVars()
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001369 self.reset_mtime_caches()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001370
Andrew Geissler5a43b432020-06-13 10:46:56 -05001371 bb_caches = bb.cache.MulticonfigCache(self.databuilder, self.data_hash, self.caches_array)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001372
Andrew Geissler5a43b432020-06-13 10:46:56 -05001373 infos = bb_caches[mc].parse(fn, self.collections[mc].get_file_appends(fn))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001374 infos = dict(infos)
1375
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001376 fn = bb.cache.realfn2virtual(fn, cls, mc)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001377 try:
1378 info_array = infos[fn]
1379 except KeyError:
1380 bb.fatal("%s does not exist" % fn)
1381
1382 if info_array[0].skipped:
1383 bb.fatal("%s was skipped: %s" % (fn, info_array[0].skipreason))
1384
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001385 self.recipecaches[mc].add_from_recipeinfo(fn, info_array)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001386
1387 # Tweak some variables
1388 item = info_array[0].pn
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001389 self.recipecaches[mc].ignored_dependencies = set()
1390 self.recipecaches[mc].bbfile_priority[fn] = 1
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001391 self.configuration.limited_deps = True
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001392
1393 # Remove external dependencies
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001394 self.recipecaches[mc].task_deps[fn]['depends'] = {}
1395 self.recipecaches[mc].deps[fn] = []
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001396 self.recipecaches[mc].rundeps[fn] = defaultdict(list)
1397 self.recipecaches[mc].runrecs[fn] = defaultdict(list)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001398
1399 # Invalidate task for target if force mode active
1400 if self.configuration.force:
1401 logger.verbose("Invalidate task %s, %s", task, fn)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001402 bb.parse.siggen.invalidate_task(task, self.recipecaches[mc], fn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001403
1404 # Setup taskdata structure
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001405 taskdata = {}
1406 taskdata[mc] = bb.taskdata.TaskData(self.configuration.abort)
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001407 taskdata[mc].add_provider(self.databuilder.mcdata[mc], self.recipecaches[mc], item)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001408
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001409 if quietlog:
1410 rqloglevel = bb.runqueue.logger.getEffectiveLevel()
1411 bb.runqueue.logger.setLevel(logging.WARNING)
1412
1413 buildname = self.databuilder.mcdata[mc].getVar("BUILDNAME")
1414 if fireevents:
1415 bb.event.fire(bb.event.BuildStarted(buildname, [item]), self.databuilder.mcdata[mc])
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001416
1417 # Execute the runqueue
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001418 runlist = [[mc, item, task, fn]]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001419
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001420 rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001421
1422 def buildFileIdle(server, rq, abort):
1423
1424 msg = None
1425 interrupted = 0
1426 if abort or self.state == state.forceshutdown:
1427 rq.finish_runqueue(True)
1428 msg = "Forced shutdown"
1429 interrupted = 2
1430 elif self.state == state.shutdown:
1431 rq.finish_runqueue(False)
1432 msg = "Stopped build"
1433 interrupted = 1
1434 failures = 0
1435 try:
1436 retval = rq.execute_runqueue()
1437 except runqueue.TaskFailure as exc:
1438 failures += len(exc.args)
1439 retval = False
1440 except SystemExit as exc:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001441 self.command.finishAsyncCommand(str(exc))
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001442 if quietlog:
1443 bb.runqueue.logger.setLevel(rqloglevel)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001444 return False
1445
1446 if not retval:
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001447 if fireevents:
1448 bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runtaskentries), buildname, item, failures, interrupted), self.databuilder.mcdata[mc])
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001449 self.command.finishAsyncCommand(msg)
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001450 # We trashed self.recipecaches above
1451 self.parsecache_valid = False
1452 self.configuration.limited_deps = False
1453 bb.parse.siggen.reset(self.data)
1454 if quietlog:
1455 bb.runqueue.logger.setLevel(rqloglevel)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001456 return False
1457 if retval is True:
1458 return True
1459 return retval
1460
Andrew Geissler635e0e42020-08-21 15:58:33 -05001461 self.idleCallBackRegister(buildFileIdle, rq)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001462
1463 def buildTargets(self, targets, task):
1464 """
1465 Attempt to build the targets specified
1466 """
1467
1468 def buildTargetsIdle(server, rq, abort):
1469 msg = None
1470 interrupted = 0
1471 if abort or self.state == state.forceshutdown:
1472 rq.finish_runqueue(True)
1473 msg = "Forced shutdown"
1474 interrupted = 2
1475 elif self.state == state.shutdown:
1476 rq.finish_runqueue(False)
1477 msg = "Stopped build"
1478 interrupted = 1
1479 failures = 0
1480 try:
1481 retval = rq.execute_runqueue()
1482 except runqueue.TaskFailure as exc:
1483 failures += len(exc.args)
1484 retval = False
1485 except SystemExit as exc:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001486 self.command.finishAsyncCommand(str(exc))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001487 return False
1488
1489 if not retval:
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001490 try:
1491 for mc in self.multiconfigs:
1492 bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runtaskentries), buildname, targets, failures, interrupted), self.databuilder.mcdata[mc])
1493 finally:
1494 self.command.finishAsyncCommand(msg)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001495 return False
1496 if retval is True:
1497 return True
1498 return retval
1499
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001500 self.reset_mtime_caches()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001501 self.buildSetVars()
1502
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001503 # If we are told to do the None task then query the default task
Andrew Geissler82c905d2020-04-13 13:39:40 -05001504 if task is None:
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001505 task = self.configuration.cmd
1506
1507 if not task.startswith("do_"):
1508 task = "do_%s" % task
1509
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001510 packages = [target if ':' in target else '%s:%s' % (target, task) for target in targets]
1511
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001512 bb.event.fire(bb.event.BuildInit(packages), self.data)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001513
1514 taskdata, runlist = self.buildTaskData(targets, task, self.configuration.abort)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001515
1516 buildname = self.data.getVar("BUILDNAME", False)
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001517
1518 # make targets to always look as <target>:do_<task>
1519 ntargets = []
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001520 for target in runlist:
1521 if target[0]:
Brad Bishop15ae2502019-06-18 21:44:24 -04001522 ntargets.append("mc:%s:%s:%s" % (target[0], target[1], target[2]))
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001523 ntargets.append("%s:%s" % (target[1], target[2]))
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001524
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001525 for mc in self.multiconfigs:
1526 bb.event.fire(bb.event.BuildStarted(buildname, ntargets), self.databuilder.mcdata[mc])
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001527
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001528 rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001529 if 'universe' in targets:
1530 rq.rqdata.warn_multi_bb = True
1531
Andrew Geissler635e0e42020-08-21 15:58:33 -05001532 self.idleCallBackRegister(buildTargetsIdle, rq)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001533
1534
1535 def getAllKeysWithFlags(self, flaglist):
1536 dump = {}
1537 for k in self.data.keys():
1538 try:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001539 expand = True
1540 flags = self.data.getVarFlags(k)
1541 if flags and "func" in flags and "python" in flags:
1542 expand = False
1543 v = self.data.getVar(k, expand)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001544 if not k.startswith("__") and not isinstance(v, bb.data_smart.DataSmart):
1545 dump[k] = {
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001546 'v' : str(v) ,
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001547 'history' : self.data.varhistory.variable(k),
1548 }
1549 for d in flaglist:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001550 if flags and d in flags:
1551 dump[k][d] = flags[d]
1552 else:
1553 dump[k][d] = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001554 except Exception as e:
1555 print(e)
1556 return dump
1557
1558
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001559 def updateCacheSync(self):
1560 if self.state == state.running:
1561 return
1562
1563 # reload files for which we got notifications
1564 for p in self.inotify_modified_files:
1565 bb.parse.update_cache(p)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001566 if p in bb.parse.BBHandler.cached_statements:
1567 del bb.parse.BBHandler.cached_statements[p]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001568 self.inotify_modified_files = []
1569
1570 if not self.baseconfig_valid:
Andrew Geisslerd1e89492021-02-12 15:35:20 -06001571 logger.debug("Reloading base configuration data")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001572 self.initConfigurationData()
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001573 self.handlePRServ()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001574
1575 # This is called for all async commands when self.state != running
1576 def updateCache(self):
1577 if self.state == state.running:
1578 return
1579
1580 if self.state in (state.shutdown, state.forceshutdown, state.error):
1581 if hasattr(self.parser, 'shutdown'):
1582 self.parser.shutdown(clean=False, force = True)
Andrew Geisslerc9f78652020-09-18 14:11:35 -05001583 self.parser.final_cleanup()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001584 raise bb.BBHandledException()
1585
1586 if self.state != state.parsing:
1587 self.updateCacheSync()
1588
1589 if self.state != state.parsing and not self.parsecache_valid:
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001590 bb.parse.siggen.reset(self.data)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001591 self.parseConfiguration ()
1592 if CookerFeatures.SEND_SANITYEVENTS in self.featureset:
Brad Bishop37a0e4d2017-12-04 01:01:44 -05001593 for mc in self.multiconfigs:
1594 bb.event.fire(bb.event.SanityCheck(False), self.databuilder.mcdata[mc])
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001595
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001596 for mc in self.multiconfigs:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001597 ignore = self.databuilder.mcdata[mc].getVar("ASSUME_PROVIDED") or ""
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001598 self.recipecaches[mc].ignored_dependencies = set(ignore.split())
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001599
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001600 for dep in self.configuration.extra_assume_provided:
1601 self.recipecaches[mc].ignored_dependencies.add(dep)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001602
Andrew Geissler5a43b432020-06-13 10:46:56 -05001603 self.collections = {}
1604
1605 mcfilelist = {}
1606 total_masked = 0
1607 searchdirs = set()
1608 for mc in self.multiconfigs:
1609 self.collections[mc] = CookerCollectFiles(self.bbfile_config_priorities, mc)
1610 (filelist, masked, search) = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc])
1611
1612 mcfilelist[mc] = filelist
1613 total_masked += masked
1614 searchdirs |= set(search)
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001615
1616 # Add inotify watches for directories searched for bb/bbappend files
1617 for dirent in searchdirs:
1618 self.add_filewatch([[dirent]], dirs=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001619
Andrew Geissler5a43b432020-06-13 10:46:56 -05001620 self.parser = CookerParser(self, mcfilelist, total_masked)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001621 self.parsecache_valid = True
1622
1623 self.state = state.parsing
1624
1625 if not self.parser.parse_next():
1626 collectlog.debug(1, "parsing complete")
1627 if self.parser.error:
1628 raise bb.BBHandledException()
1629 self.show_appends_with_no_recipes()
1630 self.handlePrefProviders()
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001631 for mc in self.multiconfigs:
Andrew Geisslerb7d28612020-07-24 16:15:54 -05001632 self.recipecaches[mc].bbfile_priority = self.collections[mc].collection_priorities(self.recipecaches[mc].pkg_fn, self.parser.mcfilelist[mc], self.data)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001633 self.state = state.running
1634
1635 # Send an event listing all stamps reachable after parsing
1636 # which the metadata may use to clean up stale data
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001637 for mc in self.multiconfigs:
1638 event = bb.event.ReachableStamps(self.recipecaches[mc].stamp)
1639 bb.event.fire(event, self.databuilder.mcdata[mc])
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001640 return None
1641
1642 return True
1643
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001644 def checkPackages(self, pkgs_to_build, task=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001645
1646 # Return a copy, don't modify the original
1647 pkgs_to_build = pkgs_to_build[:]
1648
1649 if len(pkgs_to_build) == 0:
1650 raise NothingToBuild
1651
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001652 ignore = (self.data.getVar("ASSUME_PROVIDED") or "").split()
Andrew Geisslerb7d28612020-07-24 16:15:54 -05001653 for pkg in pkgs_to_build.copy():
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001654 if pkg in ignore:
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001655 parselog.warning("Explicit target \"%s\" is in ASSUME_PROVIDED, ignoring" % pkg)
Brad Bishop15ae2502019-06-18 21:44:24 -04001656 if pkg.startswith("multiconfig:"):
1657 pkgs_to_build.remove(pkg)
1658 pkgs_to_build.append(pkg.replace("multiconfig:", "mc:"))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001659
1660 if 'world' in pkgs_to_build:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001661 pkgs_to_build.remove('world')
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001662 for mc in self.multiconfigs:
1663 bb.providers.buildWorldTargetList(self.recipecaches[mc], task)
1664 for t in self.recipecaches[mc].world_target:
1665 if mc:
Brad Bishop15ae2502019-06-18 21:44:24 -04001666 t = "mc:" + mc + ":" + t
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001667 pkgs_to_build.append(t)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001668
1669 if 'universe' in pkgs_to_build:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001670 parselog.verbnote("The \"universe\" target is only intended for testing and may produce errors.")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001671 parselog.debug(1, "collating packages for \"universe\"")
1672 pkgs_to_build.remove('universe')
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001673 for mc in self.multiconfigs:
1674 for t in self.recipecaches[mc].universe_target:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001675 if task:
1676 foundtask = False
1677 for provider_fn in self.recipecaches[mc].providers[t]:
1678 if task in self.recipecaches[mc].task_deps[provider_fn]['tasks']:
1679 foundtask = True
1680 break
1681 if not foundtask:
1682 bb.debug(1, "Skipping %s for universe tasks as task %s doesn't exist" % (t, task))
1683 continue
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001684 if mc:
Brad Bishop15ae2502019-06-18 21:44:24 -04001685 t = "mc:" + mc + ":" + t
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001686 pkgs_to_build.append(t)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001687
1688 return pkgs_to_build
1689
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001690 def pre_serve(self):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001691 return
1692
1693 def post_serve(self):
Andrew Geisslerc9f78652020-09-18 14:11:35 -05001694 self.shutdown(force=True)
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001695 prserv.serv.auto_shutdown()
Brad Bishop08902b02019-08-20 09:16:51 -04001696 if self.hashserv:
1697 self.hashserv.process.terminate()
1698 self.hashserv.process.join()
Andrew Geisslerc3d88e42020-10-02 09:45:00 -05001699 if hasattr(self, "data"):
1700 bb.event.fire(CookerExit(), self.data)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001701
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001702 def shutdown(self, force = False):
1703 if force:
1704 self.state = state.forceshutdown
1705 else:
1706 self.state = state.shutdown
1707
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001708 if self.parser:
1709 self.parser.shutdown(clean=not force, force=force)
Andrew Geisslerc9f78652020-09-18 14:11:35 -05001710 self.parser.final_cleanup()
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001711
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001712 def finishcommand(self):
1713 self.state = state.initial
1714
1715 def reset(self):
1716 self.initConfigurationData()
Brad Bishop08902b02019-08-20 09:16:51 -04001717 self.handlePRServ()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001718
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001719 def clientComplete(self):
1720 """Called when the client is done using the server"""
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001721 self.finishcommand()
1722 self.extraconfigdata = {}
1723 self.command.reset()
Andrew Geisslerc9f78652020-09-18 14:11:35 -05001724 if hasattr(self, "data"):
1725 self.databuilder.reset()
1726 self.data = self.databuilder.data
Andrew Geissler82c905d2020-04-13 13:39:40 -05001727 self.parsecache_valid = False
1728 self.baseconfig_valid = False
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001729
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001730
1731class CookerExit(bb.event.Event):
1732 """
1733 Notify clients of the Cooker shutdown
1734 """
1735
1736 def __init__(self):
1737 bb.event.Event.__init__(self)
1738
1739
1740class CookerCollectFiles(object):
Andrew Geissler5a43b432020-06-13 10:46:56 -05001741 def __init__(self, priorities, mc=''):
1742 self.mc = mc
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001743 self.bbappends = []
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001744 # Priorities is a list of tupples, with the second element as the pattern.
1745 # We need to sort the list with the longest pattern first, and so on to
1746 # the shortest. This allows nested layers to be properly evaluated.
1747 self.bbfile_config_priorities = sorted(priorities, key=lambda tup: tup[1], reverse=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001748
Andrew Geisslerb7d28612020-07-24 16:15:54 -05001749 def calc_bbfile_priority(self, filename):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001750 for _, _, regex, pri in self.bbfile_config_priorities:
1751 if regex.match(filename):
Andrew Geisslerb7d28612020-07-24 16:15:54 -05001752 return pri, regex
1753 return 0, None
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001754
1755 def get_bbfiles(self):
1756 """Get list of default .bb files by reading out the current directory"""
1757 path = os.getcwd()
1758 contents = os.listdir(path)
1759 bbfiles = []
1760 for f in contents:
1761 if f.endswith(".bb"):
1762 bbfiles.append(os.path.abspath(os.path.join(path, f)))
1763 return bbfiles
1764
1765 def find_bbfiles(self, path):
1766 """Find all the .bb and .bbappend files in a directory"""
1767 found = []
1768 for dir, dirs, files in os.walk(path):
1769 for ignored in ('SCCS', 'CVS', '.svn'):
1770 if ignored in dirs:
1771 dirs.remove(ignored)
1772 found += [os.path.join(dir, f) for f in files if (f.endswith(['.bb', '.bbappend']))]
1773
1774 return found
1775
1776 def collect_bbfiles(self, config, eventdata):
1777 """Collect all available .bb build files"""
1778 masked = 0
1779
1780 collectlog.debug(1, "collecting .bb files")
1781
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001782 files = (config.getVar( "BBFILES") or "").split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001783
1784 # Sort files by priority
Andrew Geisslerb7d28612020-07-24 16:15:54 -05001785 files.sort( key=lambda fileitem: self.calc_bbfile_priority(fileitem)[0] )
Andrew Geisslerc9f78652020-09-18 14:11:35 -05001786 config.setVar("BBFILES_PRIORITIZED", " ".join(files))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001787
1788 if not len(files):
1789 files = self.get_bbfiles()
1790
1791 if not len(files):
1792 collectlog.error("no recipe files to build, check your BBPATH and BBFILES?")
1793 bb.event.fire(CookerExit(), eventdata)
1794
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001795 # We need to track where we look so that we can add inotify watches. There
1796 # is no nice way to do this, this is horrid. We intercept the os.listdir()
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001797 # (or os.scandir() for python 3.6+) calls while we run glob().
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001798 origlistdir = os.listdir
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001799 if hasattr(os, 'scandir'):
1800 origscandir = os.scandir
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001801 searchdirs = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001802
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001803 def ourlistdir(d):
1804 searchdirs.append(d)
1805 return origlistdir(d)
1806
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001807 def ourscandir(d):
1808 searchdirs.append(d)
1809 return origscandir(d)
1810
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001811 os.listdir = ourlistdir
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001812 if hasattr(os, 'scandir'):
1813 os.scandir = ourscandir
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001814 try:
1815 # Can't use set here as order is important
1816 newfiles = []
1817 for f in files:
1818 if os.path.isdir(f):
1819 dirfiles = self.find_bbfiles(f)
1820 for g in dirfiles:
1821 if g not in newfiles:
1822 newfiles.append(g)
1823 else:
1824 globbed = glob.glob(f)
1825 if not globbed and os.path.exists(f):
1826 globbed = [f]
1827 # glob gives files in order on disk. Sort to be deterministic.
1828 for g in sorted(globbed):
1829 if g not in newfiles:
1830 newfiles.append(g)
1831 finally:
1832 os.listdir = origlistdir
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001833 if hasattr(os, 'scandir'):
1834 os.scandir = origscandir
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001835
1836 bbmask = config.getVar('BBMASK')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001837
1838 if bbmask:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001839 # First validate the individual regular expressions and ignore any
1840 # that do not compile
1841 bbmasks = []
1842 for mask in bbmask.split():
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001843 # When constructing an older style single regex, it's possible for BBMASK
1844 # to end up beginning with '|', which matches and masks _everything_.
1845 if mask.startswith("|"):
Andrew Geissler82c905d2020-04-13 13:39:40 -05001846 collectlog.warning("BBMASK contains regular expression beginning with '|', fixing: %s" % mask)
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001847 mask = mask[1:]
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001848 try:
1849 re.compile(mask)
1850 bbmasks.append(mask)
1851 except sre_constants.error:
1852 collectlog.critical("BBMASK contains an invalid regular expression, ignoring: %s" % mask)
1853
1854 # Then validate the combined regular expressions. This should never
1855 # fail, but better safe than sorry...
1856 bbmask = "|".join(bbmasks)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001857 try:
1858 bbmask_compiled = re.compile(bbmask)
1859 except sre_constants.error:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001860 collectlog.critical("BBMASK is not a valid regular expression, ignoring: %s" % bbmask)
1861 bbmask = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001862
1863 bbfiles = []
1864 bbappend = []
1865 for f in newfiles:
1866 if bbmask and bbmask_compiled.search(f):
1867 collectlog.debug(1, "skipping masked file %s", f)
1868 masked += 1
1869 continue
1870 if f.endswith('.bb'):
1871 bbfiles.append(f)
1872 elif f.endswith('.bbappend'):
1873 bbappend.append(f)
1874 else:
1875 collectlog.debug(1, "skipping %s: unknown file extension", f)
1876
1877 # Build a list of .bbappend files for each .bb file
1878 for f in bbappend:
1879 base = os.path.basename(f).replace('.bbappend', '.bb')
1880 self.bbappends.append((base, f))
1881
1882 # Find overlayed recipes
1883 # bbfiles will be in priority order which makes this easy
1884 bbfile_seen = dict()
1885 self.overlayed = defaultdict(list)
1886 for f in reversed(bbfiles):
1887 base = os.path.basename(f)
1888 if base not in bbfile_seen:
1889 bbfile_seen[base] = f
1890 else:
1891 topfile = bbfile_seen[base]
1892 self.overlayed[topfile].append(f)
1893
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001894 return (bbfiles, masked, searchdirs)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001895
1896 def get_file_appends(self, fn):
1897 """
1898 Returns a list of .bbappend files to apply to fn
1899 """
1900 filelist = []
1901 f = os.path.basename(fn)
1902 for b in self.bbappends:
1903 (bbappend, filename) = b
1904 if (bbappend == f) or ('%' in bbappend and bbappend.startswith(f[:bbappend.index('%')])):
1905 filelist.append(filename)
Andrew Geissler5a43b432020-06-13 10:46:56 -05001906 return tuple(filelist)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001907
Andrew Geisslerb7d28612020-07-24 16:15:54 -05001908 def collection_priorities(self, pkgfns, fns, d):
1909 # Return the priorities of the entries in pkgfns
1910 # Also check that all the regexes in self.bbfile_config_priorities are used
1911 # (but to do that we need to ensure skipped recipes aren't counted, nor
1912 # collections in BBFILE_PATTERN_IGNORE_EMPTY)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001913
1914 priorities = {}
Andrew Geisslerb7d28612020-07-24 16:15:54 -05001915 seen = set()
1916 matched = set()
1917
1918 matched_regex = set()
1919 unmatched_regex = set()
1920 for _, _, regex, _ in self.bbfile_config_priorities:
1921 unmatched_regex.add(regex)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001922
1923 # Calculate priorities for each file
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001924 for p in pkgfns:
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001925 realfn, cls, mc = bb.cache.virtualfn2realfn(p)
Andrew Geisslerb7d28612020-07-24 16:15:54 -05001926 priorities[p], regex = self.calc_bbfile_priority(realfn)
1927 if regex in unmatched_regex:
1928 matched_regex.add(regex)
1929 unmatched_regex.remove(regex)
1930 seen.add(realfn)
1931 if regex:
1932 matched.add(realfn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001933
Andrew Geisslerb7d28612020-07-24 16:15:54 -05001934 if unmatched_regex:
1935 # Account for bbappend files
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001936 for b in self.bbappends:
1937 (bbfile, append) = b
Andrew Geisslerb7d28612020-07-24 16:15:54 -05001938 seen.add(append)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001939
Andrew Geisslerb7d28612020-07-24 16:15:54 -05001940 # Account for skipped recipes
1941 seen.update(fns)
1942
1943 seen.difference_update(matched)
1944
1945 def already_matched(fn):
1946 for regex in matched_regex:
1947 if regex.match(fn):
1948 return True
1949 return False
1950
1951 for unmatch in unmatched_regex.copy():
1952 for fn in seen:
1953 if unmatch.match(fn):
1954 # If the bbappend or file was already matched by another regex, skip it
1955 # e.g. for a layer within a layer, the outer regex could match, the inner
1956 # regex may match nothing and we should warn about that
1957 if already_matched(fn):
1958 continue
1959 unmatched_regex.remove(unmatch)
1960 break
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001961
1962 for collection, pattern, regex, _ in self.bbfile_config_priorities:
Andrew Geisslerb7d28612020-07-24 16:15:54 -05001963 if regex in unmatched_regex:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001964 if d.getVar('BBFILE_PATTERN_IGNORE_EMPTY_%s' % collection) != '1':
Andrew Geissler5a43b432020-06-13 10:46:56 -05001965 collectlog.warning("No bb files in %s matched BBFILE_PATTERN_%s '%s'" % (self.mc if self.mc else 'default',
1966 collection, pattern))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001967
1968 return priorities
1969
1970class ParsingFailure(Exception):
1971 def __init__(self, realexception, recipe):
1972 self.realexception = realexception
1973 self.recipe = recipe
1974 Exception.__init__(self, realexception, recipe)
1975
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001976class Parser(multiprocessing.Process):
1977 def __init__(self, jobs, results, quit, init, profile):
1978 self.jobs = jobs
1979 self.results = results
1980 self.quit = quit
1981 self.init = init
1982 multiprocessing.Process.__init__(self)
1983 self.context = bb.utils.get_context().copy()
1984 self.handlers = bb.event.get_class_handlers().copy()
1985 self.profile = profile
1986
1987 def run(self):
1988
1989 if not self.profile:
1990 self.realrun()
1991 return
1992
1993 try:
1994 import cProfile as profile
1995 except:
1996 import profile
1997 prof = profile.Profile()
1998 try:
1999 profile.Profile.runcall(prof, self.realrun)
2000 finally:
2001 logfile = "profile-parse-%s.log" % multiprocessing.current_process().name
2002 prof.dump_stats(logfile)
2003
2004 def realrun(self):
2005 if self.init:
2006 self.init()
2007
2008 pending = []
2009 while True:
2010 try:
2011 self.quit.get_nowait()
Patrick Williamsc0f7c042017-02-23 20:41:17 -06002012 except queue.Empty:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002013 pass
2014 else:
Andrew Geisslerc9f78652020-09-18 14:11:35 -05002015 self.results.close()
2016 self.results.join_thread()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002017 break
2018
2019 if pending:
2020 result = pending.pop()
2021 else:
2022 try:
Brad Bishop19323692019-04-05 15:28:33 -04002023 job = self.jobs.pop()
2024 except IndexError:
Andrew Geisslerc9f78652020-09-18 14:11:35 -05002025 self.results.close()
2026 self.results.join_thread()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002027 break
2028 result = self.parse(*job)
Andrew Geissler82c905d2020-04-13 13:39:40 -05002029 # Clear the siggen cache after parsing to control memory usage, its huge
2030 bb.parse.siggen.postparsing_clean_cache()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002031 try:
2032 self.results.put(result, timeout=0.25)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06002033 except queue.Full:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002034 pending.append(result)
2035
Andrew Geissler5a43b432020-06-13 10:46:56 -05002036 def parse(self, mc, cache, filename, appends):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002037 try:
Andrew Geissler82c905d2020-04-13 13:39:40 -05002038 origfilter = bb.event.LogHandler.filter
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05002039 # Record the filename we're parsing into any events generated
2040 def parse_filter(self, record):
2041 record.taskpid = bb.event.worker_pid
2042 record.fn = filename
2043 return True
2044
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002045 # Reset our environment and handlers to the original settings
2046 bb.utils.set_context(self.context.copy())
2047 bb.event.set_class_handlers(self.handlers.copy())
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05002048 bb.event.LogHandler.filter = parse_filter
2049
Andrew Geissler5a43b432020-06-13 10:46:56 -05002050 return True, mc, cache.parse(filename, appends)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002051 except Exception as exc:
2052 tb = sys.exc_info()[2]
2053 exc.recipe = filename
2054 exc.traceback = list(bb.exceptions.extract_traceback(tb, context=3))
2055 return True, exc
2056 # Need to turn BaseExceptions into Exceptions here so we gracefully shutdown
2057 # and for example a worker thread doesn't just exit on its own in response to
2058 # a SystemExit event for example.
2059 except BaseException as exc:
2060 return True, ParsingFailure(exc, filename)
Andrew Geissler82c905d2020-04-13 13:39:40 -05002061 finally:
2062 bb.event.LogHandler.filter = origfilter
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002063
2064class CookerParser(object):
Andrew Geissler5a43b432020-06-13 10:46:56 -05002065 def __init__(self, cooker, mcfilelist, masked):
2066 self.mcfilelist = mcfilelist
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002067 self.cooker = cooker
2068 self.cfgdata = cooker.data
2069 self.cfghash = cooker.data_hash
Patrick Williamsc0f7c042017-02-23 20:41:17 -06002070 self.cfgbuilder = cooker.databuilder
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002071
2072 # Accounting statistics
2073 self.parsed = 0
2074 self.cached = 0
2075 self.error = 0
2076 self.masked = masked
2077
2078 self.skipped = 0
2079 self.virtuals = 0
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002080
2081 self.current = 0
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002082 self.process_names = []
2083
Andrew Geissler5a43b432020-06-13 10:46:56 -05002084 self.bb_caches = bb.cache.MulticonfigCache(self.cfgbuilder, self.cfghash, cooker.caches_array)
2085 self.fromcache = set()
2086 self.willparse = set()
2087 for mc in self.cooker.multiconfigs:
2088 for filename in self.mcfilelist[mc]:
2089 appends = self.cooker.collections[mc].get_file_appends(filename)
2090 if not self.bb_caches[mc].cacheValid(filename, appends):
2091 self.willparse.add((mc, self.bb_caches[mc], filename, appends))
2092 else:
2093 self.fromcache.add((mc, self.bb_caches[mc], filename, appends))
2094
2095 self.total = len(self.fromcache) + len(self.willparse)
2096 self.toparse = len(self.willparse)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06002097 self.progress_chunk = int(max(self.toparse / 100, 1))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002098
Brad Bishop6e60e8b2018-02-01 10:27:11 -05002099 self.num_processes = min(int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS") or
Andrew Geissler5a43b432020-06-13 10:46:56 -05002100 multiprocessing.cpu_count()), self.toparse)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05002101
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002102 self.start()
2103 self.haveshutdown = False
Andrew Geisslerc9f78652020-09-18 14:11:35 -05002104 self.syncthread = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002105
2106 def start(self):
2107 self.results = self.load_cached()
2108 self.processes = []
2109 if self.toparse:
2110 bb.event.fire(bb.event.ParseStarted(self.toparse), self.cfgdata)
2111 def init():
Andrew Geisslerc9f78652020-09-18 14:11:35 -05002112 signal.signal(signal.SIGTERM, signal.SIG_DFL)
2113 signal.signal(signal.SIGHUP, signal.SIG_DFL)
2114 signal.signal(signal.SIGINT, signal.SIG_IGN)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05002115 bb.utils.set_process_name(multiprocessing.current_process().name)
2116 multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, exitpriority=1)
2117 multiprocessing.util.Finalize(None, bb.fetch.fetcher_parse_save, exitpriority=1)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002118
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002119 self.parser_quit = multiprocessing.Queue(maxsize=self.num_processes)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002120 self.result_queue = multiprocessing.Queue()
Brad Bishop19323692019-04-05 15:28:33 -04002121
2122 def chunkify(lst,n):
2123 return [lst[i::n] for i in range(n)]
Andrew Geissler5a43b432020-06-13 10:46:56 -05002124 self.jobs = chunkify(list(self.willparse), self.num_processes)
Brad Bishop19323692019-04-05 15:28:33 -04002125
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002126 for i in range(0, self.num_processes):
Brad Bishop19323692019-04-05 15:28:33 -04002127 parser = Parser(self.jobs[i], self.result_queue, self.parser_quit, init, self.cooker.configuration.profile)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002128 parser.start()
2129 self.process_names.append(parser.name)
2130 self.processes.append(parser)
2131
2132 self.results = itertools.chain(self.results, self.parse_generator())
2133
2134 def shutdown(self, clean=True, force=False):
2135 if not self.toparse:
2136 return
2137 if self.haveshutdown:
2138 return
2139 self.haveshutdown = True
2140
2141 if clean:
2142 event = bb.event.ParseCompleted(self.cached, self.parsed,
2143 self.skipped, self.masked,
2144 self.virtuals, self.error,
2145 self.total)
2146
2147 bb.event.fire(event, self.cfgdata)
Andrew Geisslerc9f78652020-09-18 14:11:35 -05002148
2149 for process in self.processes:
2150 self.parser_quit.put(None)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002151
Brad Bishop08902b02019-08-20 09:16:51 -04002152 # Cleanup the queue before call process.join(), otherwise there might be
2153 # deadlocks.
2154 while True:
2155 try:
2156 self.result_queue.get(timeout=0.25)
2157 except queue.Empty:
2158 break
2159
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002160 for process in self.processes:
2161 if force:
2162 process.join(.1)
2163 process.terminate()
2164 else:
2165 process.join()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002166
Andrew Geisslerc9f78652020-09-18 14:11:35 -05002167 self.parser_quit.close()
2168 # Allow data left in the cancel queue to be discarded
2169 self.parser_quit.cancel_join_thread()
2170
Andrew Geissler5a43b432020-06-13 10:46:56 -05002171 def sync_caches():
2172 for c in self.bb_caches.values():
2173 c.sync()
2174
Andrew Geisslerc9f78652020-09-18 14:11:35 -05002175 sync = threading.Thread(target=sync_caches, name="SyncThread")
2176 self.syncthread = sync
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002177 sync.start()
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05002178 bb.codeparser.parser_cache_savemerge()
2179 bb.fetch.fetcher_parse_done()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002180 if self.cooker.configuration.profile:
2181 profiles = []
2182 for i in self.process_names:
2183 logfile = "profile-parse-%s.log" % i
2184 if os.path.exists(logfile):
2185 profiles.append(logfile)
2186
2187 pout = "profile-parse.log.processed"
2188 bb.utils.process_profilelog(profiles, pout = pout)
2189 print("Processed parsing statistics saved to %s" % (pout))
2190
Andrew Geisslerc9f78652020-09-18 14:11:35 -05002191 def final_cleanup(self):
2192 if self.syncthread:
2193 self.syncthread.join()
2194
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002195 def load_cached(self):
Andrew Geissler5a43b432020-06-13 10:46:56 -05002196 for mc, cache, filename, appends in self.fromcache:
2197 cached, infos = cache.load(filename, appends)
2198 yield not cached, mc, infos
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002199
2200 def parse_generator(self):
2201 while True:
2202 if self.parsed >= self.toparse:
2203 break
2204
2205 try:
2206 result = self.result_queue.get(timeout=0.25)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06002207 except queue.Empty:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002208 pass
2209 else:
2210 value = result[1]
2211 if isinstance(value, BaseException):
2212 raise value
2213 else:
2214 yield result
2215
2216 def parse_next(self):
2217 result = []
2218 parsed = None
2219 try:
Andrew Geissler5a43b432020-06-13 10:46:56 -05002220 parsed, mc, result = next(self.results)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002221 except StopIteration:
2222 self.shutdown()
2223 return False
2224 except bb.BBHandledException as exc:
2225 self.error += 1
2226 logger.error('Failed to parse recipe: %s' % exc.recipe)
Andrew Geissler90fd73c2021-03-05 15:25:55 -06002227 self.shutdown(clean=False, force=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002228 return False
2229 except ParsingFailure as exc:
2230 self.error += 1
2231 logger.error('Unable to parse %s: %s' %
2232 (exc.recipe, bb.exceptions.to_string(exc.realexception)))
Andrew Geissler90fd73c2021-03-05 15:25:55 -06002233 self.shutdown(clean=False, force=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002234 return False
2235 except bb.parse.ParseError as exc:
2236 self.error += 1
2237 logger.error(str(exc))
Andrew Geissler90fd73c2021-03-05 15:25:55 -06002238 self.shutdown(clean=False, force=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002239 return False
2240 except bb.data_smart.ExpansionError as exc:
2241 self.error += 1
Patrick Williamsc0f7c042017-02-23 20:41:17 -06002242 bbdir = os.path.dirname(__file__) + os.sep
2243 etype, value, _ = sys.exc_info()
2244 tb = list(itertools.dropwhile(lambda e: e.filename.startswith(bbdir), exc.traceback))
2245 logger.error('ExpansionError during parsing %s', value.recipe,
2246 exc_info=(etype, value, tb))
Andrew Geissler90fd73c2021-03-05 15:25:55 -06002247 self.shutdown(clean=False, force=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002248 return False
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002249 except Exception as exc:
2250 self.error += 1
2251 etype, value, tb = sys.exc_info()
2252 if hasattr(value, "recipe"):
Patrick Williamsc0f7c042017-02-23 20:41:17 -06002253 logger.error('Unable to parse %s' % value.recipe,
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002254 exc_info=(etype, value, exc.traceback))
2255 else:
2256 # Most likely, an exception occurred during raising an exception
2257 import traceback
2258 logger.error('Exception during parse: %s' % traceback.format_exc())
Andrew Geissler90fd73c2021-03-05 15:25:55 -06002259 self.shutdown(clean=False, force=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002260 return False
2261
2262 self.current += 1
2263 self.virtuals += len(result)
2264 if parsed:
2265 self.parsed += 1
2266 if self.parsed % self.progress_chunk == 0:
2267 bb.event.fire(bb.event.ParseProgress(self.parsed, self.toparse),
2268 self.cfgdata)
2269 else:
2270 self.cached += 1
2271
2272 for virtualfn, info_array in result:
2273 if info_array[0].skipped:
2274 self.skipped += 1
2275 self.cooker.skiplist[virtualfn] = SkippedPackage(info_array[0])
Andrew Geissler5a43b432020-06-13 10:46:56 -05002276 self.bb_caches[mc].add_info(virtualfn, info_array, self.cooker.recipecaches[mc],
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002277 parsed=parsed, watcher = self.cooker.add_filewatch)
2278 return True
2279
2280 def reparse(self, filename):
Andrew Geissler5a43b432020-06-13 10:46:56 -05002281 to_reparse = set()
2282 for mc in self.cooker.multiconfigs:
2283 to_reparse.add((mc, filename, self.cooker.collections[mc].get_file_appends(filename)))
2284
2285 for mc, filename, appends in to_reparse:
2286 infos = self.bb_caches[mc].parse(filename, appends)
2287 for vfn, info_array in infos:
2288 self.cooker.recipecaches[mc].add_from_recipeinfo(vfn, info_array)