blob: 1b6ee3032cd0e198be007e7c68a325064205cfa8 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001#
2# Copyright (C) 2003, 2004 Chris Larson
3# Copyright (C) 2003, 2004 Phil Blundell
4# Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
5# Copyright (C) 2005 Holger Hans Peter Freyther
6# Copyright (C) 2005 ROAD GmbH
7# Copyright (C) 2006 - 2007 Richard Purdie
8#
Brad Bishopc342db32019-05-15 21:57:59 -04009# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -050010#
Patrick Williamsc0f7c042017-02-23 20:41:17 -060011
Patrick Williamsc124f4f2015-09-15 14:41:29 -050012import sys, os, glob, os.path, re, time
Patrick Williamsc124f4f2015-09-15 14:41:29 -050013import itertools
14import logging
15import multiprocessing
Patrick Williamsc124f4f2015-09-15 14:41:29 -050016import threading
Patrick Williamsc0f7c042017-02-23 20:41:17 -060017from io import StringIO, UnsupportedOperation
Patrick Williamsc124f4f2015-09-15 14:41:29 -050018from contextlib import closing
Patrick Williamsc0f7c042017-02-23 20:41:17 -060019from collections import defaultdict, namedtuple
Patrick Williamsc124f4f2015-09-15 14:41:29 -050020import bb, bb.exceptions, bb.command
21from bb import utils, data, parse, event, cache, providers, taskdata, runqueue, build
Patrick Williamsc0f7c042017-02-23 20:41:17 -060022import queue
Patrick Williamsc124f4f2015-09-15 14:41:29 -050023import signal
Patrick Williamsc124f4f2015-09-15 14:41:29 -050024import prserv.serv
25import pyinotify
Patrick Williamsc0f7c042017-02-23 20:41:17 -060026import json
27import pickle
28import codecs
Brad Bishop08902b02019-08-20 09:16:51 -040029import hashserv
Patrick Williamsc124f4f2015-09-15 14:41:29 -050030
31logger = logging.getLogger("BitBake")
32collectlog = logging.getLogger("BitBake.Collection")
33buildlog = logging.getLogger("BitBake.Build")
34parselog = logging.getLogger("BitBake.Parsing")
35providerlog = logging.getLogger("BitBake.Provider")
36
37class NoSpecificMatch(bb.BBHandledException):
38 """
39 Exception raised when no or multiple file matches are found
40 """
41
42class NothingToBuild(Exception):
43 """
44 Exception raised when there is nothing to build
45 """
46
47class CollectionError(bb.BBHandledException):
48 """
49 Exception raised when layer configuration is incorrect
50 """
51
52class state:
Patrick Williamsc0f7c042017-02-23 20:41:17 -060053 initial, parsing, running, shutdown, forceshutdown, stopped, error = list(range(7))
Patrick Williamsc124f4f2015-09-15 14:41:29 -050054
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050055 @classmethod
56 def get_name(cls, code):
57 for name in dir(cls):
58 value = getattr(cls, name)
59 if type(value) == type(cls.initial) and value == code:
60 return name
61 raise ValueError("Invalid status code: %s" % code)
62
Patrick Williamsc124f4f2015-09-15 14:41:29 -050063
64class SkippedPackage:
65 def __init__(self, info = None, reason = None):
66 self.pn = None
67 self.skipreason = None
68 self.provides = None
69 self.rprovides = None
70
71 if info:
72 self.pn = info.pn
73 self.skipreason = info.skipreason
74 self.provides = info.provides
Andrew Geisslerd1e89492021-02-12 15:35:20 -060075 self.rprovides = info.packages + info.rprovides
76 for package in info.packages:
77 self.rprovides += info.rprovides_pkg[package]
Patrick Williamsc124f4f2015-09-15 14:41:29 -050078 elif reason:
79 self.skipreason = reason
80
81
82class CookerFeatures(object):
Patrick Williamsc0f7c042017-02-23 20:41:17 -060083 _feature_list = [HOB_EXTRA_CACHES, BASEDATASTORE_TRACKING, SEND_SANITYEVENTS] = list(range(3))
Patrick Williamsc124f4f2015-09-15 14:41:29 -050084
85 def __init__(self):
86 self._features=set()
87
88 def setFeature(self, f):
89 # validate we got a request for a feature we support
90 if f not in CookerFeatures._feature_list:
91 return
92 self._features.add(f)
93
94 def __contains__(self, f):
95 return f in self._features
96
97 def __iter__(self):
98 return self._features.__iter__()
99
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600100 def __next__(self):
101 return next(self._features)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500102
103
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600104class EventWriter:
105 def __init__(self, cooker, eventfile):
106 self.file_inited = None
107 self.cooker = cooker
108 self.eventfile = eventfile
109 self.event_queue = []
110
111 def write_event(self, event):
112 with open(self.eventfile, "a") as f:
113 try:
114 str_event = codecs.encode(pickle.dumps(event), 'base64').decode('utf-8')
115 f.write("%s\n" % json.dumps({"class": event.__module__ + "." + event.__class__.__name__,
116 "vars": str_event}))
117 except Exception as err:
118 import traceback
119 print(err, traceback.format_exc())
120
121 def send(self, event):
122 if self.file_inited:
123 # we have the file, just write the event
124 self.write_event(event)
125 else:
126 # init on bb.event.BuildStarted
127 name = "%s.%s" % (event.__module__, event.__class__.__name__)
128 if name in ("bb.event.BuildStarted", "bb.cooker.CookerExit"):
129 with open(self.eventfile, "w") as f:
130 f.write("%s\n" % json.dumps({ "allvariables" : self.cooker.getAllKeysWithFlags(["doc", "func"])}))
131
132 self.file_inited = True
133
134 # write pending events
135 for evt in self.event_queue:
136 self.write_event(evt)
137
138 # also write the current event
139 self.write_event(event)
140 else:
141 # queue all events until the file is inited
142 self.event_queue.append(event)
143
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500144#============================================================================#
145# BBCooker
146#============================================================================#
147class BBCooker:
148 """
149 Manages one bitbake build run
150 """
151
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500152 def __init__(self, featureSet=None, idleCallBackRegister=None):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600153 self.recipecaches = None
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500154 self.eventlog = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500155 self.skiplist = {}
156 self.featureset = CookerFeatures()
157 if featureSet:
158 for f in featureSet:
159 self.featureset.setFeature(f)
160
Patrick Williams45852732022-04-02 08:58:32 -0500161 self.orig_syspath = sys.path.copy()
162 self.orig_sysmodules = [*sys.modules]
163
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500164 self.configuration = bb.cookerdata.CookerConfiguration()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500165
Andrew Geissler635e0e42020-08-21 15:58:33 -0500166 self.idleCallBackRegister = idleCallBackRegister
167
Brad Bishopf058f492019-01-28 23:50:33 -0500168 bb.debug(1, "BBCooker starting %s" % time.time())
169 sys.stdout.flush()
170
Patrick Williamsde0582f2022-04-08 10:23:27 -0500171 self.configwatcher = None
172 self.confignotifier = None
Brad Bishopf058f492019-01-28 23:50:33 -0500173
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500174 self.watchmask = pyinotify.IN_CLOSE_WRITE | pyinotify.IN_CREATE | pyinotify.IN_DELETE | \
175 pyinotify.IN_DELETE_SELF | pyinotify.IN_MODIFY | pyinotify.IN_MOVE_SELF | \
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500176 pyinotify.IN_MOVED_FROM | pyinotify.IN_MOVED_TO
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500177
Patrick Williamsde0582f2022-04-08 10:23:27 -0500178 self.watcher = None
179 self.notifier = None
Brad Bishopf058f492019-01-28 23:50:33 -0500180
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500181 # If being called by something like tinfoil, we need to clean cached data
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500182 # which may now be invalid
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500183 bb.parse.clear_cache()
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500184 bb.parse.BBHandler.cached_statements = {}
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500185
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500186 self.ui_cmdline = None
Brad Bishop08902b02019-08-20 09:16:51 -0400187 self.hashserv = None
Brad Bishopa34c0302019-09-23 22:34:48 -0400188 self.hashservaddr = None
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500189
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500190 self.inotify_modified_files = []
191
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000192 def _process_inotify_updates(server, cooker, halt):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500193 cooker.process_inotify_updates()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500194 return 1.0
195
Andrew Geissler635e0e42020-08-21 15:58:33 -0500196 self.idleCallBackRegister(_process_inotify_updates, self)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500197
198 # TOSTOP must not be set or our children will hang when they output
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600199 try:
200 fd = sys.stdout.fileno()
201 if os.isatty(fd):
202 import termios
203 tcattr = termios.tcgetattr(fd)
204 if tcattr[3] & termios.TOSTOP:
205 buildlog.info("The terminal had the TOSTOP bit set, clearing...")
206 tcattr[3] = tcattr[3] & ~termios.TOSTOP
207 termios.tcsetattr(fd, termios.TCSANOW, tcattr)
208 except UnsupportedOperation:
209 pass
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500210
211 self.command = bb.command.Command(self)
212 self.state = state.initial
213
214 self.parser = None
215
216 signal.signal(signal.SIGTERM, self.sigterm_exception)
217 # Let SIGHUP exit as SIGTERM
218 signal.signal(signal.SIGHUP, self.sigterm_exception)
219
Brad Bishopf058f492019-01-28 23:50:33 -0500220 bb.debug(1, "BBCooker startup complete %s" % time.time())
221 sys.stdout.flush()
222
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500223 def init_configdata(self):
224 if not hasattr(self, "data"):
225 self.initConfigurationData()
226 bb.debug(1, "BBCooker parsed base configuration %s" % time.time())
227 sys.stdout.flush()
228 self.handlePRServ()
229
Patrick Williamsde0582f2022-04-08 10:23:27 -0500230 def setupConfigWatcher(self):
231 if self.configwatcher:
232 self.configwatcher.close()
233 self.confignotifier = None
234 self.configwatcher = None
235 self.configwatcher = pyinotify.WatchManager()
236 self.configwatcher.bbseen = set()
237 self.configwatcher.bbwatchedfiles = set()
238 self.confignotifier = pyinotify.Notifier(self.configwatcher, self.config_notifications)
239
240 def setupParserWatcher(self):
241 if self.watcher:
242 self.watcher.close()
243 self.notifier = None
244 self.watcher = None
245 self.watcher = pyinotify.WatchManager()
246 self.watcher.bbseen = set()
247 self.watcher.bbwatchedfiles = set()
248 self.notifier = pyinotify.Notifier(self.watcher, self.notifications)
249
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500250 def process_inotify_updates(self):
251 for n in [self.confignotifier, self.notifier]:
Patrick Williamsde0582f2022-04-08 10:23:27 -0500252 if n and n.check_events(timeout=0):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500253 # read notified events and enqeue them
254 n.read_events()
255 n.process_events()
256
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500257 def config_notifications(self, event):
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500258 if event.maskname == "IN_Q_OVERFLOW":
259 bb.warn("inotify event queue overflowed, invalidating caches.")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500260 self.parsecache_valid = False
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500261 self.baseconfig_valid = False
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500262 bb.parse.clear_cache()
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500263 return
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500264 if not event.pathname in self.configwatcher.bbwatchedfiles:
265 return
Andrew Geissler9aee5002022-03-30 16:27:02 +0000266 if "IN_ISDIR" in event.maskname:
Patrick Williams45852732022-04-02 08:58:32 -0500267 if "IN_CREATE" in event.maskname or "IN_DELETE" in event.maskname:
268 if event.pathname in self.configwatcher.bbseen:
269 self.configwatcher.bbseen.remove(event.pathname)
270 # Could remove all entries starting with the directory but for now...
271 bb.parse.clear_cache()
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500272 if not event.pathname in self.inotify_modified_files:
273 self.inotify_modified_files.append(event.pathname)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500274 self.baseconfig_valid = False
275
276 def notifications(self, event):
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500277 if event.maskname == "IN_Q_OVERFLOW":
278 bb.warn("inotify event queue overflowed, invalidating caches.")
279 self.parsecache_valid = False
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500280 bb.parse.clear_cache()
281 return
282 if event.pathname.endswith("bitbake-cookerdaemon.log") \
283 or event.pathname.endswith("bitbake.lock"):
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500284 return
Andrew Geissler9aee5002022-03-30 16:27:02 +0000285 if "IN_ISDIR" in event.maskname:
Patrick Williams45852732022-04-02 08:58:32 -0500286 if "IN_CREATE" in event.maskname or "IN_DELETE" in event.maskname:
287 if event.pathname in self.watcher.bbseen:
288 self.watcher.bbseen.remove(event.pathname)
289 # Could remove all entries starting with the directory but for now...
290 bb.parse.clear_cache()
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500291 if not event.pathname in self.inotify_modified_files:
292 self.inotify_modified_files.append(event.pathname)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500293 self.parsecache_valid = False
294
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500295 def add_filewatch(self, deps, watcher=None, dirs=False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500296 if not watcher:
297 watcher = self.watcher
298 for i in deps:
Andrew Geissler82c905d2020-04-13 13:39:40 -0500299 watcher.bbwatchedfiles.add(i[0])
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500300 if dirs:
301 f = i[0]
302 else:
303 f = os.path.dirname(i[0])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500304 if f in watcher.bbseen:
305 continue
Andrew Geissler82c905d2020-04-13 13:39:40 -0500306 watcher.bbseen.add(f)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500307 watchtarget = None
308 while True:
309 # We try and add watches for files that don't exist but if they did, would influence
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500310 # the parser. The parent directory of these files may not exist, in which case we need
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500311 # to watch any parent that does exist for changes.
312 try:
313 watcher.add_watch(f, self.watchmask, quiet=False)
314 if watchtarget:
Andrew Geissler82c905d2020-04-13 13:39:40 -0500315 watcher.bbwatchedfiles.add(watchtarget)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500316 break
317 except pyinotify.WatchManagerError as e:
318 if 'ENOENT' in str(e):
319 watchtarget = f
320 f = os.path.dirname(f)
321 if f in watcher.bbseen:
322 break
Andrew Geissler82c905d2020-04-13 13:39:40 -0500323 watcher.bbseen.add(f)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500324 continue
325 if 'ENOSPC' in str(e):
326 providerlog.error("No space left on device or exceeds fs.inotify.max_user_watches?")
327 providerlog.error("To check max_user_watches: sysctl -n fs.inotify.max_user_watches.")
328 providerlog.error("To modify max_user_watches: sysctl -n -w fs.inotify.max_user_watches=<value>.")
329 providerlog.error("Root privilege is required to modify max_user_watches.")
330 raise
331
332 def sigterm_exception(self, signum, stackframe):
333 if signum == signal.SIGTERM:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500334 bb.warn("Cooker received SIGTERM, shutting down...")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500335 elif signum == signal.SIGHUP:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500336 bb.warn("Cooker received SIGHUP, shutting down...")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500337 self.state = state.forceshutdown
338
339 def setFeatures(self, features):
340 # we only accept a new feature set if we're in state initial, so we can reset without problems
341 if not self.state in [state.initial, state.shutdown, state.forceshutdown, state.stopped, state.error]:
342 raise Exception("Illegal state for feature set change")
343 original_featureset = list(self.featureset)
344 for feature in features:
345 self.featureset.setFeature(feature)
346 bb.debug(1, "Features set %s (was %s)" % (original_featureset, list(self.featureset)))
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500347 if (original_featureset != list(self.featureset)) and self.state != state.error and hasattr(self, "data"):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500348 self.reset()
349
350 def initConfigurationData(self):
351
352 self.state = state.initial
353 self.caches_array = []
354
Patrick Williams45852732022-04-02 08:58:32 -0500355 sys.path = self.orig_syspath.copy()
356 for mod in [*sys.modules]:
357 if mod not in self.orig_sysmodules:
358 del sys.modules[mod]
359
Patrick Williamsde0582f2022-04-08 10:23:27 -0500360 self.setupConfigWatcher()
361
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500362 # Need to preserve BB_CONSOLELOG over resets
363 consolelog = None
364 if hasattr(self, "data"):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500365 consolelog = self.data.getVar("BB_CONSOLELOG")
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500366
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500367 if CookerFeatures.BASEDATASTORE_TRACKING in self.featureset:
368 self.enableDataTracking()
369
370 all_extra_cache_names = []
371 # We hardcode all known cache types in a single place, here.
372 if CookerFeatures.HOB_EXTRA_CACHES in self.featureset:
373 all_extra_cache_names.append("bb.cache_extra:HobRecipeInfo")
374
375 caches_name_array = ['bb.cache:CoreRecipeInfo'] + all_extra_cache_names
376
377 # At least CoreRecipeInfo will be loaded, so caches_array will never be empty!
378 # This is the entry point, no further check needed!
379 for var in caches_name_array:
380 try:
381 module_name, cache_name = var.split(':')
382 module = __import__(module_name, fromlist=(cache_name,))
383 self.caches_array.append(getattr(module, cache_name))
384 except ImportError as exc:
385 logger.critical("Unable to import extra RecipeInfo '%s' from '%s': %s" % (cache_name, module_name, exc))
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500386 raise bb.BBHandledException()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500387
388 self.databuilder = bb.cookerdata.CookerDataBuilder(self.configuration, False)
389 self.databuilder.parseBaseConfiguration()
390 self.data = self.databuilder.data
391 self.data_hash = self.databuilder.data_hash
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500392 self.extraconfigdata = {}
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500393
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500394 if consolelog:
395 self.data.setVar("BB_CONSOLELOG", consolelog)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500396
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500397 self.data.setVar('BB_CMDLINE', self.ui_cmdline)
398
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500399 if CookerFeatures.BASEDATASTORE_TRACKING in self.featureset:
400 self.disableDataTracking()
401
Brad Bishop15ae2502019-06-18 21:44:24 -0400402 for mc in self.databuilder.mcdata.values():
403 mc.renameVar("__depends", "__base_depends")
404 self.add_filewatch(mc.getVar("__base_depends", False), self.configwatcher)
Patrick Williams92b42cb2022-09-03 06:53:57 -0500405 mc.setVar("__bbclasstype", "recipe")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500406
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500407 self.baseconfig_valid = True
408 self.parsecache_valid = False
409
410 def handlePRServ(self):
411 # Setup a PR Server based on the new configuration
412 try:
413 self.prhost = prserv.serv.auto_start(self.data)
414 except prserv.serv.PRServiceConfigError as e:
Andrew Geisslerd159c7f2021-09-02 21:05:58 -0500415 bb.fatal("Unable to start PR Server, exiting, check the bitbake-cookerdaemon.log")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500416
Brad Bishopa34c0302019-09-23 22:34:48 -0400417 if self.data.getVar("BB_HASHSERVE") == "auto":
418 # Create a new hash server bound to a unix domain socket
Brad Bishop08902b02019-08-20 09:16:51 -0400419 if not self.hashserv:
420 dbfile = (self.data.getVar("PERSISTENT_DIR") or self.data.getVar("CACHE")) + "/hashserv.db"
Andrew Geissler595f6302022-01-24 19:11:47 +0000421 upstream = self.data.getVar("BB_HASHSERVE_UPSTREAM") or None
422 if upstream:
423 import socket
424 try:
425 sock = socket.create_connection(upstream.split(":"), 5)
426 sock.close()
427 except socket.error as e:
428 bb.warn("BB_HASHSERVE_UPSTREAM is not valid, unable to connect hash equivalence server at '%s': %s"
429 % (upstream, repr(e)))
430
Brad Bishopa34c0302019-09-23 22:34:48 -0400431 self.hashservaddr = "unix://%s/hashserve.sock" % self.data.getVar("TOPDIR")
Andrew Geissler5199d832021-09-24 16:47:35 -0500432 self.hashserv = hashserv.create_server(
433 self.hashservaddr,
434 dbfile,
435 sync=False,
Andrew Geissler595f6302022-01-24 19:11:47 +0000436 upstream=upstream,
Andrew Geissler5199d832021-09-24 16:47:35 -0500437 )
Patrick Williams213cb262021-08-07 19:21:33 -0500438 self.hashserv.serve_as_process()
Brad Bishopa34c0302019-09-23 22:34:48 -0400439 self.data.setVar("BB_HASHSERVE", self.hashservaddr)
440 self.databuilder.origdata.setVar("BB_HASHSERVE", self.hashservaddr)
441 self.databuilder.data.setVar("BB_HASHSERVE", self.hashservaddr)
Brad Bishop08902b02019-08-20 09:16:51 -0400442 for mc in self.databuilder.mcdata:
Brad Bishopa34c0302019-09-23 22:34:48 -0400443 self.databuilder.mcdata[mc].setVar("BB_HASHSERVE", self.hashservaddr)
Brad Bishop08902b02019-08-20 09:16:51 -0400444
445 bb.parse.init_parser(self.data)
446
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500447 def enableDataTracking(self):
448 self.configuration.tracking = True
449 if hasattr(self, "data"):
450 self.data.enableTracking()
451
452 def disableDataTracking(self):
453 self.configuration.tracking = False
454 if hasattr(self, "data"):
455 self.data.disableTracking()
456
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500457 def parseConfiguration(self):
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600458 self.updateCacheSync()
459
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500460 # Change nice level if we're asked to
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500461 nice = self.data.getVar("BB_NICE_LEVEL")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500462 if nice:
463 curnice = os.nice(0)
464 nice = int(nice) - curnice
465 buildlog.verbose("Renice to %s " % os.nice(nice))
466
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600467 if self.recipecaches:
468 del self.recipecaches
469 self.multiconfigs = self.databuilder.mcdata.keys()
470 self.recipecaches = {}
471 for mc in self.multiconfigs:
472 self.recipecaches[mc] = bb.cache.CacheData(self.caches_array)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500473
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500474 self.handleCollections(self.data.getVar("BBFILE_COLLECTIONS"))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500475
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500476 self.parsecache_valid = False
477
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500478 def updateConfigOpts(self, options, environment, cmdline):
479 self.ui_cmdline = cmdline
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500480 clean = True
481 for o in options:
482 if o in ['prefile', 'postfile']:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500483 # Only these options may require a reparse
484 try:
485 if getattr(self.configuration, o) == options[o]:
486 # Value is the same, no need to mark dirty
487 continue
488 except AttributeError:
489 pass
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600490 logger.debug("Marking as dirty due to '%s' option change to '%s'" % (o, options[o]))
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500491 print("Marking as dirty due to '%s' option change to '%s'" % (o, options[o]))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500492 clean = False
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500493 if hasattr(self.configuration, o):
494 setattr(self.configuration, o, options[o])
495
496 if self.configuration.writeeventlog:
497 if self.eventlog and self.eventlog[0] != self.configuration.writeeventlog:
498 bb.event.unregister_UIHhandler(self.eventlog[1])
499 if not self.eventlog or self.eventlog[0] != self.configuration.writeeventlog:
500 # we log all events to a file if so directed
501 # register the log file writer as UI Handler
502 writer = EventWriter(self, self.configuration.writeeventlog)
503 EventLogWriteHandler = namedtuple('EventLogWriteHandler', ['event'])
504 self.eventlog = (self.configuration.writeeventlog, bb.event.register_UIHhandler(EventLogWriteHandler(writer)))
505
506 bb.msg.loggerDefaultLogLevel = self.configuration.default_loglevel
507 bb.msg.loggerDefaultDomains = self.configuration.debug_domains
508
509 if hasattr(self, "data"):
510 origenv = bb.data.init()
511 for k in environment:
512 origenv.setVar(k, environment[k])
513 self.data.setVar("BB_ORIGENV", origenv)
514
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500515 for k in bb.utils.approved_variables():
516 if k in environment and k not in self.configuration.env:
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600517 logger.debug("Updating new environment variable %s to %s" % (k, environment[k]))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500518 self.configuration.env[k] = environment[k]
519 clean = False
520 if k in self.configuration.env and k not in environment:
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600521 logger.debug("Updating environment variable %s (deleted)" % (k))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500522 del self.configuration.env[k]
523 clean = False
524 if k not in self.configuration.env and k not in environment:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500525 continue
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500526 if environment[k] != self.configuration.env[k]:
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600527 logger.debug("Updating environment variable %s from %s to %s" % (k, self.configuration.env[k], environment[k]))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500528 self.configuration.env[k] = environment[k]
529 clean = False
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500530
531 # Now update all the variables not in the datastore to match
532 self.configuration.env = environment
533
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500534 if not clean:
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600535 logger.debug("Base environment change, triggering reparse")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500536 self.reset()
537
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000538 def runCommands(self, server, data, halt):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500539 """
540 Run any queued asynchronous command
541 This is done by the idle handler so it runs in true context rather than
542 tied to any UI.
543 """
544
545 return self.command.runAsyncCommand()
546
547 def showVersions(self):
548
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500549 (latest_versions, preferred_versions, required) = self.findProviders()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500550
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500551 logger.plain("%-35s %25s %25s %25s", "Recipe Name", "Latest Version", "Preferred Version", "Required Version")
552 logger.plain("%-35s %25s %25s %25s\n", "===========", "==============", "=================", "================")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500553
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500554 for p in sorted(self.recipecaches[''].pkg_pn):
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500555 preferred = preferred_versions[p]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500556 latest = latest_versions[p]
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500557 requiredstr = ""
558 preferredstr = ""
559 if required[p]:
560 if preferred[0] is not None:
561 requiredstr = preferred[0][0] + ":" + preferred[0][1] + '-' + preferred[0][2]
562 else:
563 bb.fatal("REQUIRED_VERSION of package %s not available" % p)
564 else:
565 preferredstr = preferred[0][0] + ":" + preferred[0][1] + '-' + preferred[0][2]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500566
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500567 lateststr = latest[0][0] + ":" + latest[0][1] + "-" + latest[0][2]
568
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500569 if preferred == latest:
570 preferredstr = ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500571
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500572 logger.plain("%-35s %25s %25s %25s", p, lateststr, preferredstr, requiredstr)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500573
574 def showEnvironment(self, buildfile=None, pkgs_to_build=None):
575 """
576 Show the outer or per-recipe environment
577 """
578 fn = None
579 envdata = None
Brad Bishop15ae2502019-06-18 21:44:24 -0400580 mc = ''
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500581 if not pkgs_to_build:
582 pkgs_to_build = []
583
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500584 orig_tracking = self.configuration.tracking
585 if not orig_tracking:
586 self.enableDataTracking()
587 self.reset()
Andrew Geissler9aee5002022-03-30 16:27:02 +0000588 # reset() resets to the UI requested value so we have to redo this
589 self.enableDataTracking()
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500590
Brad Bishop15ae2502019-06-18 21:44:24 -0400591 def mc_base(p):
592 if p.startswith('mc:'):
593 s = p.split(':')
594 if len(s) == 2:
595 return s[1]
596 return None
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500597
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500598 if buildfile:
599 # Parse the configuration here. We need to do it explicitly here since
600 # this showEnvironment() code path doesn't use the cache
601 self.parseConfiguration()
602
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600603 fn, cls, mc = bb.cache.virtualfn2realfn(buildfile)
Andrew Geissler5a43b432020-06-13 10:46:56 -0500604 fn = self.matchFile(fn, mc)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600605 fn = bb.cache.realfn2virtual(fn, cls, mc)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500606 elif len(pkgs_to_build) == 1:
Brad Bishop15ae2502019-06-18 21:44:24 -0400607 mc = mc_base(pkgs_to_build[0])
608 if not mc:
609 ignore = self.data.getVar("ASSUME_PROVIDED") or ""
610 if pkgs_to_build[0] in set(ignore.split()):
611 bb.fatal("%s is in ASSUME_PROVIDED" % pkgs_to_build[0])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500612
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000613 taskdata, runlist = self.buildTaskData(pkgs_to_build, None, self.configuration.halt, allowincomplete=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500614
Brad Bishop15ae2502019-06-18 21:44:24 -0400615 mc = runlist[0][0]
616 fn = runlist[0][3]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500617
618 if fn:
619 try:
Andrew Geissler5a43b432020-06-13 10:46:56 -0500620 bb_caches = bb.cache.MulticonfigCache(self.databuilder, self.data_hash, self.caches_array)
621 envdata = bb_caches[mc].loadDataFull(fn, self.collections[mc].get_file_appends(fn))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500622 except Exception as e:
623 parselog.exception("Unable to read %s", fn)
624 raise
Brad Bishop15ae2502019-06-18 21:44:24 -0400625 else:
626 if not mc in self.databuilder.mcdata:
627 bb.fatal('Not multiconfig named "%s" found' % mc)
628 envdata = self.databuilder.mcdata[mc]
629 data.expandKeys(envdata)
630 parse.ast.runAnonFuncs(envdata)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500631
632 # Display history
633 with closing(StringIO()) as env:
634 self.data.inchistory.emit(env)
635 logger.plain(env.getvalue())
636
637 # emit variables and shell functions
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500638 with closing(StringIO()) as env:
639 data.emit_env(env, envdata, True)
640 logger.plain(env.getvalue())
641
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000642 # emit the metadata which isn't valid shell
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500643 for e in sorted(envdata.keys()):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600644 if envdata.getVarFlag(e, 'func', False) and envdata.getVarFlag(e, 'python', False):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500645 logger.plain("\npython %s () {\n%s}\n", e, envdata.getVar(e, False))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500646
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500647 if not orig_tracking:
648 self.disableDataTracking()
649 self.reset()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500650
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000651 def buildTaskData(self, pkgs_to_build, task, halt, allowincomplete=False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500652 """
653 Prepare a runqueue and taskdata object for iteration over pkgs_to_build
654 """
655 bb.event.fire(bb.event.TreeDataPreparationStarted(), self.data)
656
657 # A task of None means use the default task
658 if task is None:
659 task = self.configuration.cmd
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500660 if not task.startswith("do_"):
661 task = "do_%s" % task
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500662
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500663 targetlist = self.checkPackages(pkgs_to_build, task)
664 fulltargetlist = []
665 defaulttask_implicit = ''
666 defaulttask_explicit = False
667 wildcard = False
668
669 # Wild card expansion:
Brad Bishop15ae2502019-06-18 21:44:24 -0400670 # Replace string such as "mc:*:bash"
671 # into "mc:A:bash mc:B:bash bash"
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500672 for k in targetlist:
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600673 if k.startswith("mc:") and k.count(':') >= 2:
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500674 if wildcard:
675 bb.fatal('multiconfig conflict')
676 if k.split(":")[1] == "*":
677 wildcard = True
678 for mc in self.multiconfigs:
679 if mc:
680 fulltargetlist.append(k.replace('*', mc))
681 # implicit default task
682 else:
683 defaulttask_implicit = k.split(":")[2]
684 else:
685 fulltargetlist.append(k)
686 else:
687 defaulttask_explicit = True
688 fulltargetlist.append(k)
689
690 if not defaulttask_explicit and defaulttask_implicit != '':
691 fulltargetlist.append(defaulttask_implicit)
692
693 bb.debug(1,"Target list: %s" % (str(fulltargetlist)))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600694 taskdata = {}
695 localdata = {}
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500696
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600697 for mc in self.multiconfigs:
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000698 taskdata[mc] = bb.taskdata.TaskData(halt, skiplist=self.skiplist, allowincomplete=allowincomplete)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600699 localdata[mc] = data.createCopy(self.databuilder.mcdata[mc])
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600700 bb.data.expandKeys(localdata[mc])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500701
702 current = 0
703 runlist = []
704 for k in fulltargetlist:
Andrew Geisslerb7d28612020-07-24 16:15:54 -0500705 origk = k
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600706 mc = ""
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600707 if k.startswith("mc:") and k.count(':') >= 2:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600708 mc = k.split(":")[1]
709 k = ":".join(k.split(":")[2:])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500710 ktask = task
711 if ":do_" in k:
712 k2 = k.split(":do_")
713 k = k2[0]
714 ktask = k2[1]
Andrew Geisslerb7d28612020-07-24 16:15:54 -0500715
716 if mc not in self.multiconfigs:
717 bb.fatal("Multiconfig dependency %s depends on nonexistent multiconfig configuration named %s" % (origk, mc))
718
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600719 taskdata[mc].add_provider(localdata[mc], self.recipecaches[mc], k)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500720 current += 1
721 if not ktask.startswith("do_"):
722 ktask = "do_%s" % ktask
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600723 if k not in taskdata[mc].build_targets or not taskdata[mc].build_targets[k]:
724 # e.g. in ASSUME_PROVIDED
725 continue
726 fn = taskdata[mc].build_targets[k][0]
727 runlist.append([mc, k, ktask, fn])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500728 bb.event.fire(bb.event.TreeDataPreparationProgress(current, len(fulltargetlist)), self.data)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600729
Brad Bishop15ae2502019-06-18 21:44:24 -0400730 havemc = False
731 for mc in self.multiconfigs:
732 if taskdata[mc].get_mcdepends():
733 havemc = True
Brad Bishopf058f492019-01-28 23:50:33 -0500734
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800735 # No need to do check providers if there are no mcdeps or not an mc build
Brad Bishop15ae2502019-06-18 21:44:24 -0400736 if havemc or len(self.multiconfigs) > 1:
Andrew Geissler99467da2019-02-25 18:54:23 -0600737 seen = set()
738 new = True
739 # Make sure we can provide the multiconfig dependency
740 while new:
741 mcdeps = set()
742 # Add unresolved first, so we can get multiconfig indirect dependencies on time
743 for mc in self.multiconfigs:
744 taskdata[mc].add_unresolved(localdata[mc], self.recipecaches[mc])
745 mcdeps |= set(taskdata[mc].get_mcdepends())
746 new = False
Patrick Williams03907ee2022-05-01 06:28:52 -0500747 for k in mcdeps:
748 if k in seen:
749 continue
750 l = k.split(':')
751 depmc = l[2]
752 if depmc not in self.multiconfigs:
753 bb.fatal("Multiconfig dependency %s depends on nonexistent multiconfig configuration named configuration %s" % (k,depmc))
754 else:
755 logger.debug("Adding providers for multiconfig dependency %s" % l[3])
756 taskdata[depmc].add_provider(localdata[depmc], self.recipecaches[depmc], l[3])
757 seen.add(k)
758 new = True
Brad Bishopf058f492019-01-28 23:50:33 -0500759
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600760 for mc in self.multiconfigs:
761 taskdata[mc].add_unresolved(localdata[mc], self.recipecaches[mc])
762
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500763 bb.event.fire(bb.event.TreeDataPreparationCompleted(len(fulltargetlist)), self.data)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600764 return taskdata, runlist
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500765
766 def prepareTreeData(self, pkgs_to_build, task):
767 """
768 Prepare a runqueue and taskdata object for iteration over pkgs_to_build
769 """
770
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000771 # We set halt to False here to prevent unbuildable targets raising
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500772 # an exception when we're just generating data
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600773 taskdata, runlist = self.buildTaskData(pkgs_to_build, task, False, allowincomplete=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500774
775 return runlist, taskdata
776
777 ######## WARNING : this function requires cache_extra to be enabled ########
778
779 def generateTaskDepTreeData(self, pkgs_to_build, task):
780 """
781 Create a dependency graph of pkgs_to_build including reverse dependency
782 information.
783 """
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500784 if not task.startswith("do_"):
785 task = "do_%s" % task
786
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500787 runlist, taskdata = self.prepareTreeData(pkgs_to_build, task)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600788 rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500789 rq.rqdata.prepare()
790 return self.buildDependTree(rq, taskdata)
791
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600792 @staticmethod
793 def add_mc_prefix(mc, pn):
794 if mc:
Brad Bishop15ae2502019-06-18 21:44:24 -0400795 return "mc:%s:%s" % (mc, pn)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600796 return pn
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500797
798 def buildDependTree(self, rq, taskdata):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600799 seen_fns = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500800 depend_tree = {}
801 depend_tree["depends"] = {}
802 depend_tree["tdepends"] = {}
803 depend_tree["pn"] = {}
804 depend_tree["rdepends-pn"] = {}
805 depend_tree["packages"] = {}
806 depend_tree["rdepends-pkg"] = {}
807 depend_tree["rrecs-pkg"] = {}
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500808 depend_tree['providermap'] = {}
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600809 depend_tree["layer-priorities"] = self.bbfile_config_priorities
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500810
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600811 for mc in taskdata:
812 for name, fn in list(taskdata[mc].get_providermap().items()):
813 pn = self.recipecaches[mc].pkg_fn[fn]
814 pn = self.add_mc_prefix(mc, pn)
815 if name != pn:
816 version = "%s:%s-%s" % self.recipecaches[mc].pkg_pepvpr[fn]
817 depend_tree['providermap'][name] = (pn, version)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500818
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600819 for tid in rq.rqdata.runtaskentries:
820 (mc, fn, taskname, taskfn) = bb.runqueue.split_tid_mcfn(tid)
821 pn = self.recipecaches[mc].pkg_fn[taskfn]
822 pn = self.add_mc_prefix(mc, pn)
823 version = "%s:%s-%s" % self.recipecaches[mc].pkg_pepvpr[taskfn]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500824 if pn not in depend_tree["pn"]:
825 depend_tree["pn"][pn] = {}
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600826 depend_tree["pn"][pn]["filename"] = taskfn
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500827 depend_tree["pn"][pn]["version"] = version
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600828 depend_tree["pn"][pn]["inherits"] = self.recipecaches[mc].inherits.get(taskfn, None)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500829
830 # if we have extra caches, list all attributes they bring in
831 extra_info = []
832 for cache_class in self.caches_array:
833 if type(cache_class) is type and issubclass(cache_class, bb.cache.RecipeInfoCommon) and hasattr(cache_class, 'cachefields'):
834 cachefields = getattr(cache_class, 'cachefields', [])
835 extra_info = extra_info + cachefields
836
837 # for all attributes stored, add them to the dependency tree
838 for ei in extra_info:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600839 depend_tree["pn"][pn][ei] = vars(self.recipecaches[mc])[ei][taskfn]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500840
841
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500842 dotname = "%s.%s" % (pn, bb.runqueue.taskname_from_tid(tid))
843 if not dotname in depend_tree["tdepends"]:
844 depend_tree["tdepends"][dotname] = []
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600845 for dep in rq.rqdata.runtaskentries[tid].depends:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800846 (depmc, depfn, _, deptaskfn) = bb.runqueue.split_tid_mcfn(dep)
847 deppn = self.recipecaches[depmc].pkg_fn[deptaskfn]
Andrew Geissler595f6302022-01-24 19:11:47 +0000848 if depmc:
849 depmc = "mc:" + depmc + ":"
850 depend_tree["tdepends"][dotname].append("%s%s.%s" % (depmc, deppn, bb.runqueue.taskname_from_tid(dep)))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600851 if taskfn not in seen_fns:
852 seen_fns.append(taskfn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500853 packages = []
854
855 depend_tree["depends"][pn] = []
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600856 for dep in taskdata[mc].depids[taskfn]:
857 depend_tree["depends"][pn].append(dep)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500858
859 depend_tree["rdepends-pn"][pn] = []
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600860 for rdep in taskdata[mc].rdepids[taskfn]:
861 depend_tree["rdepends-pn"][pn].append(rdep)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500862
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600863 rdepends = self.recipecaches[mc].rundeps[taskfn]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500864 for package in rdepends:
865 depend_tree["rdepends-pkg"][package] = []
866 for rdepend in rdepends[package]:
867 depend_tree["rdepends-pkg"][package].append(rdepend)
868 packages.append(package)
869
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600870 rrecs = self.recipecaches[mc].runrecs[taskfn]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500871 for package in rrecs:
872 depend_tree["rrecs-pkg"][package] = []
873 for rdepend in rrecs[package]:
874 depend_tree["rrecs-pkg"][package].append(rdepend)
875 if not package in packages:
876 packages.append(package)
877
878 for package in packages:
879 if package not in depend_tree["packages"]:
880 depend_tree["packages"][package] = {}
881 depend_tree["packages"][package]["pn"] = pn
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600882 depend_tree["packages"][package]["filename"] = taskfn
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500883 depend_tree["packages"][package]["version"] = version
884
885 return depend_tree
886
887 ######## WARNING : this function requires cache_extra to be enabled ########
888 def generatePkgDepTreeData(self, pkgs_to_build, task):
889 """
890 Create a dependency tree of pkgs_to_build, returning the data.
891 """
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500892 if not task.startswith("do_"):
893 task = "do_%s" % task
894
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500895 _, taskdata = self.prepareTreeData(pkgs_to_build, task)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500896
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600897 seen_fns = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500898 depend_tree = {}
899 depend_tree["depends"] = {}
900 depend_tree["pn"] = {}
901 depend_tree["rdepends-pn"] = {}
902 depend_tree["rdepends-pkg"] = {}
903 depend_tree["rrecs-pkg"] = {}
904
905 # if we have extra caches, list all attributes they bring in
906 extra_info = []
907 for cache_class in self.caches_array:
908 if type(cache_class) is type and issubclass(cache_class, bb.cache.RecipeInfoCommon) and hasattr(cache_class, 'cachefields'):
909 cachefields = getattr(cache_class, 'cachefields', [])
910 extra_info = extra_info + cachefields
911
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600912 tids = []
913 for mc in taskdata:
914 for tid in taskdata[mc].taskentries:
915 tids.append(tid)
916
917 for tid in tids:
918 (mc, fn, taskname, taskfn) = bb.runqueue.split_tid_mcfn(tid)
919
920 pn = self.recipecaches[mc].pkg_fn[taskfn]
921 pn = self.add_mc_prefix(mc, pn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500922
923 if pn not in depend_tree["pn"]:
924 depend_tree["pn"][pn] = {}
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600925 depend_tree["pn"][pn]["filename"] = taskfn
926 version = "%s:%s-%s" % self.recipecaches[mc].pkg_pepvpr[taskfn]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500927 depend_tree["pn"][pn]["version"] = version
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600928 rdepends = self.recipecaches[mc].rundeps[taskfn]
929 rrecs = self.recipecaches[mc].runrecs[taskfn]
930 depend_tree["pn"][pn]["inherits"] = self.recipecaches[mc].inherits.get(taskfn, None)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500931
932 # for all extra attributes stored, add them to the dependency tree
933 for ei in extra_info:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600934 depend_tree["pn"][pn][ei] = vars(self.recipecaches[mc])[ei][taskfn]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500935
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600936 if taskfn not in seen_fns:
937 seen_fns.append(taskfn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500938
939 depend_tree["depends"][pn] = []
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500940 for dep in taskdata[mc].depids[taskfn]:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500941 pn_provider = ""
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600942 if dep in taskdata[mc].build_targets and taskdata[mc].build_targets[dep]:
943 fn_provider = taskdata[mc].build_targets[dep][0]
944 pn_provider = self.recipecaches[mc].pkg_fn[fn_provider]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500945 else:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500946 pn_provider = dep
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600947 pn_provider = self.add_mc_prefix(mc, pn_provider)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500948 depend_tree["depends"][pn].append(pn_provider)
949
950 depend_tree["rdepends-pn"][pn] = []
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600951 for rdep in taskdata[mc].rdepids[taskfn]:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500952 pn_rprovider = ""
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600953 if rdep in taskdata[mc].run_targets and taskdata[mc].run_targets[rdep]:
954 fn_rprovider = taskdata[mc].run_targets[rdep][0]
955 pn_rprovider = self.recipecaches[mc].pkg_fn[fn_rprovider]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500956 else:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600957 pn_rprovider = rdep
958 pn_rprovider = self.add_mc_prefix(mc, pn_rprovider)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500959 depend_tree["rdepends-pn"][pn].append(pn_rprovider)
960
961 depend_tree["rdepends-pkg"].update(rdepends)
962 depend_tree["rrecs-pkg"].update(rrecs)
963
964 return depend_tree
965
966 def generateDepTreeEvent(self, pkgs_to_build, task):
967 """
968 Create a task dependency graph of pkgs_to_build.
969 Generate an event with the result
970 """
971 depgraph = self.generateTaskDepTreeData(pkgs_to_build, task)
972 bb.event.fire(bb.event.DepTreeGenerated(depgraph), self.data)
973
974 def generateDotGraphFiles(self, pkgs_to_build, task):
975 """
976 Create a task dependency graph of pkgs_to_build.
977 Save the result to a set of .dot files.
978 """
979
980 depgraph = self.generateTaskDepTreeData(pkgs_to_build, task)
981
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500982 with open('pn-buildlist', 'w') as f:
983 for pn in depgraph["pn"]:
984 f.write(pn + "\n")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500985 logger.info("PN build list saved to 'pn-buildlist'")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500986
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500987 # Remove old format output files to ensure no confusion with stale data
988 try:
989 os.unlink('pn-depends.dot')
990 except FileNotFoundError:
991 pass
992 try:
993 os.unlink('package-depends.dot')
994 except FileNotFoundError:
995 pass
Brad Bishop79641f22019-09-10 07:20:22 -0400996 try:
997 os.unlink('recipe-depends.dot')
998 except FileNotFoundError:
999 pass
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001000
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001001 with open('task-depends.dot', 'w') as f:
1002 f.write("digraph depends {\n")
Brad Bishop316dfdd2018-06-25 12:45:53 -04001003 for task in sorted(depgraph["tdepends"]):
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001004 (pn, taskname) = task.rsplit(".", 1)
1005 fn = depgraph["pn"][pn]["filename"]
1006 version = depgraph["pn"][pn]["version"]
1007 f.write('"%s.%s" [label="%s %s\\n%s\\n%s"]\n' % (pn, taskname, pn, taskname, version, fn))
Brad Bishop316dfdd2018-06-25 12:45:53 -04001008 for dep in sorted(depgraph["tdepends"][task]):
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001009 f.write('"%s" -> "%s"\n' % (task, dep))
1010 f.write("}\n")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001011 logger.info("Task dependencies saved to 'task-depends.dot'")
1012
1013 def show_appends_with_no_recipes(self):
Andrew Geissler5a43b432020-06-13 10:46:56 -05001014 appends_without_recipes = {}
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001015 # Determine which bbappends haven't been applied
Andrew Geissler5a43b432020-06-13 10:46:56 -05001016 for mc in self.multiconfigs:
1017 # First get list of recipes, including skipped
1018 recipefns = list(self.recipecaches[mc].pkg_fn.keys())
1019 recipefns.extend(self.skiplist.keys())
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001020
Andrew Geissler5a43b432020-06-13 10:46:56 -05001021 # Work out list of bbappends that have been applied
1022 applied_appends = []
1023 for fn in recipefns:
1024 applied_appends.extend(self.collections[mc].get_file_appends(fn))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001025
Andrew Geissler5a43b432020-06-13 10:46:56 -05001026 appends_without_recipes[mc] = []
1027 for _, appendfn in self.collections[mc].bbappends:
1028 if not appendfn in applied_appends:
1029 appends_without_recipes[mc].append(appendfn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001030
Andrew Geissler5a43b432020-06-13 10:46:56 -05001031 msgs = []
1032 for mc in sorted(appends_without_recipes.keys()):
1033 if appends_without_recipes[mc]:
1034 msgs.append('No recipes in %s available for:\n %s' % (mc if mc else 'default',
1035 '\n '.join(appends_without_recipes[mc])))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001036
Andrew Geissler5a43b432020-06-13 10:46:56 -05001037 if msgs:
1038 msg = "\n".join(msgs)
1039 warn_only = self.databuilder.mcdata[mc].getVar("BB_DANGLINGAPPENDS_WARNONLY", \
1040 False) or "no"
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001041 if warn_only.lower() in ("1", "yes", "true"):
1042 bb.warn(msg)
1043 else:
1044 bb.fatal(msg)
1045
1046 def handlePrefProviders(self):
1047
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001048 for mc in self.multiconfigs:
1049 localdata = data.createCopy(self.databuilder.mcdata[mc])
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001050 bb.data.expandKeys(localdata)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001051
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001052 # Handle PREFERRED_PROVIDERS
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001053 for p in (localdata.getVar('PREFERRED_PROVIDERS') or "").split():
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001054 try:
1055 (providee, provider) = p.split(':')
1056 except:
1057 providerlog.critical("Malformed option in PREFERRED_PROVIDERS variable: %s" % p)
1058 continue
1059 if providee in self.recipecaches[mc].preferred and self.recipecaches[mc].preferred[providee] != provider:
1060 providerlog.error("conflicting preferences for %s: both %s and %s specified", providee, provider, self.recipecaches[mc].preferred[providee])
1061 self.recipecaches[mc].preferred[providee] = provider
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001062
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001063 def findConfigFilePath(self, configfile):
1064 """
1065 Find the location on disk of configfile and if it exists and was parsed by BitBake
1066 emit the ConfigFilePathFound event with the path to the file.
1067 """
1068 path = bb.cookerdata.findConfigFile(configfile, self.data)
1069 if not path:
1070 return
1071
1072 # Generate a list of parsed configuration files by searching the files
1073 # listed in the __depends and __base_depends variables with a .conf suffix.
1074 conffiles = []
1075 dep_files = self.data.getVar('__base_depends', False) or []
1076 dep_files = dep_files + (self.data.getVar('__depends', False) or [])
1077
1078 for f in dep_files:
1079 if f[0].endswith(".conf"):
1080 conffiles.append(f[0])
1081
1082 _, conf, conffile = path.rpartition("conf/")
1083 match = os.path.join(conf, conffile)
1084 # Try and find matches for conf/conffilename.conf as we don't always
1085 # have the full path to the file.
1086 for cfg in conffiles:
1087 if cfg.endswith(match):
1088 bb.event.fire(bb.event.ConfigFilePathFound(path),
1089 self.data)
1090 break
1091
1092 def findFilesMatchingInDir(self, filepattern, directory):
1093 """
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001094 Searches for files containing the substring 'filepattern' which are children of
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001095 'directory' in each BBPATH. i.e. to find all rootfs package classes available
1096 to BitBake one could call findFilesMatchingInDir(self, 'rootfs_', 'classes')
1097 or to find all machine configuration files one could call:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001098 findFilesMatchingInDir(self, '.conf', 'conf/machine')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001099 """
1100
1101 matches = []
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001102 bbpaths = self.data.getVar('BBPATH').split(':')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001103 for path in bbpaths:
1104 dirpath = os.path.join(path, directory)
1105 if os.path.exists(dirpath):
1106 for root, dirs, files in os.walk(dirpath):
1107 for f in files:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001108 if filepattern in f:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001109 matches.append(f)
1110
1111 if matches:
1112 bb.event.fire(bb.event.FilesMatchingFound(filepattern, matches), self.data)
1113
Patrick Williams93c203f2021-10-06 16:15:23 -05001114 def testCookerCommandEvent(self, filepattern):
1115 # Dummy command used by OEQA selftest to test tinfoil without IO
1116 matches = ["A", "B"]
1117 bb.event.fire(bb.event.FilesMatchingFound(filepattern, matches), self.data)
1118
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001119 def findProviders(self, mc=''):
Andrew Geissler82c905d2020-04-13 13:39:40 -05001120 return bb.providers.findProviders(self.databuilder.mcdata[mc], self.recipecaches[mc], self.recipecaches[mc].pkg_pn)
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001121
1122 def findBestProvider(self, pn, mc=''):
1123 if pn in self.recipecaches[mc].providers:
1124 filenames = self.recipecaches[mc].providers[pn]
Andrew Geissler82c905d2020-04-13 13:39:40 -05001125 eligible, foundUnique = bb.providers.filterProviders(filenames, pn, self.databuilder.mcdata[mc], self.recipecaches[mc])
Andrew Geissler95ac1b82021-03-31 14:34:31 -05001126 if eligible is not None:
1127 filename = eligible[0]
1128 else:
1129 filename = None
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001130 return None, None, None, filename
1131 elif pn in self.recipecaches[mc].pkg_pn:
Andrew Geissler95ac1b82021-03-31 14:34:31 -05001132 (latest, latest_f, preferred_ver, preferred_file, required) = bb.providers.findBestProvider(pn, self.databuilder.mcdata[mc], self.recipecaches[mc], self.recipecaches[mc].pkg_pn)
1133 if required and preferred_file is None:
1134 return None, None, None, None
1135 return (latest, latest_f, preferred_ver, preferred_file)
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001136 else:
1137 return None, None, None, None
1138
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001139 def findConfigFiles(self, varname):
1140 """
1141 Find config files which are appropriate values for varname.
1142 i.e. MACHINE, DISTRO
1143 """
1144 possible = []
1145 var = varname.lower()
1146
1147 data = self.data
1148 # iterate configs
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001149 bbpaths = data.getVar('BBPATH').split(':')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001150 for path in bbpaths:
1151 confpath = os.path.join(path, "conf", var)
1152 if os.path.exists(confpath):
1153 for root, dirs, files in os.walk(confpath):
1154 # get all child files, these are appropriate values
1155 for f in files:
1156 val, sep, end = f.rpartition('.')
1157 if end == 'conf':
1158 possible.append(val)
1159
1160 if possible:
1161 bb.event.fire(bb.event.ConfigFilesFound(var, possible), self.data)
1162
1163 def findInheritsClass(self, klass):
1164 """
1165 Find all recipes which inherit the specified class
1166 """
1167 pkg_list = []
1168
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001169 for pfn in self.recipecaches[''].pkg_fn:
1170 inherits = self.recipecaches[''].inherits.get(pfn, None)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001171 if inherits and klass in inherits:
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001172 pkg_list.append(self.recipecaches[''].pkg_fn[pfn])
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001173
1174 return pkg_list
1175
1176 def generateTargetsTree(self, klass=None, pkgs=None):
1177 """
1178 Generate a dependency tree of buildable targets
1179 Generate an event with the result
1180 """
1181 # if the caller hasn't specified a pkgs list default to universe
1182 if not pkgs:
1183 pkgs = ['universe']
1184 # if inherited_class passed ensure all recipes which inherit the
1185 # specified class are included in pkgs
1186 if klass:
1187 extra_pkgs = self.findInheritsClass(klass)
1188 pkgs = pkgs + extra_pkgs
1189
1190 # generate a dependency tree for all our packages
1191 tree = self.generatePkgDepTreeData(pkgs, 'build')
1192 bb.event.fire(bb.event.TargetsTreeGenerated(tree), self.data)
1193
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001194 def interactiveMode( self ):
1195 """Drop off into a shell"""
1196 try:
1197 from bb import shell
1198 except ImportError:
1199 parselog.exception("Interactive mode not available")
Andrew Geisslerc9f78652020-09-18 14:11:35 -05001200 raise bb.BBHandledException()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001201 else:
1202 shell.start( self )
1203
1204
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001205 def handleCollections(self, collections):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001206 """Handle collections"""
1207 errors = False
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001208 self.bbfile_config_priorities = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001209 if collections:
1210 collection_priorities = {}
1211 collection_depends = {}
1212 collection_list = collections.split()
1213 min_prio = 0
1214 for c in collection_list:
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001215 bb.debug(1,'Processing %s in collection list' % (c))
1216
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001217 # Get collection priority if defined explicitly
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001218 priority = self.data.getVar("BBFILE_PRIORITY_%s" % c)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001219 if priority:
1220 try:
1221 prio = int(priority)
1222 except ValueError:
1223 parselog.error("invalid value for BBFILE_PRIORITY_%s: \"%s\"", c, priority)
1224 errors = True
1225 if min_prio == 0 or prio < min_prio:
1226 min_prio = prio
1227 collection_priorities[c] = prio
1228 else:
1229 collection_priorities[c] = None
1230
1231 # Check dependencies and store information for priority calculation
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001232 deps = self.data.getVar("LAYERDEPENDS_%s" % c)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001233 if deps:
1234 try:
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001235 depDict = bb.utils.explode_dep_versions2(deps)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001236 except bb.utils.VersionStringException as vse:
1237 bb.fatal('Error parsing LAYERDEPENDS_%s: %s' % (c, str(vse)))
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001238 for dep, oplist in list(depDict.items()):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001239 if dep in collection_list:
1240 for opstr in oplist:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001241 layerver = self.data.getVar("LAYERVERSION_%s" % dep)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001242 (op, depver) = opstr.split()
1243 if layerver:
1244 try:
1245 res = bb.utils.vercmp_string_op(layerver, depver, op)
1246 except bb.utils.VersionStringException as vse:
1247 bb.fatal('Error parsing LAYERDEPENDS_%s: %s' % (c, str(vse)))
1248 if not res:
1249 parselog.error("Layer '%s' depends on version %s of layer '%s', but version %s is currently enabled in your configuration. Check that you are using the correct matching versions/branches of these two layers.", c, opstr, dep, layerver)
1250 errors = True
1251 else:
1252 parselog.error("Layer '%s' depends on version %s of layer '%s', which exists in your configuration but does not specify a version. Check that you are using the correct matching versions/branches of these two layers.", c, opstr, dep)
1253 errors = True
1254 else:
1255 parselog.error("Layer '%s' depends on layer '%s', but this layer is not enabled in your configuration", c, dep)
1256 errors = True
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001257 collection_depends[c] = list(depDict.keys())
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001258 else:
1259 collection_depends[c] = []
1260
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001261 # Check recommends and store information for priority calculation
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001262 recs = self.data.getVar("LAYERRECOMMENDS_%s" % c)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001263 if recs:
1264 try:
1265 recDict = bb.utils.explode_dep_versions2(recs)
1266 except bb.utils.VersionStringException as vse:
1267 bb.fatal('Error parsing LAYERRECOMMENDS_%s: %s' % (c, str(vse)))
1268 for rec, oplist in list(recDict.items()):
1269 if rec in collection_list:
1270 if oplist:
1271 opstr = oplist[0]
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001272 layerver = self.data.getVar("LAYERVERSION_%s" % rec)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001273 if layerver:
1274 (op, recver) = opstr.split()
1275 try:
1276 res = bb.utils.vercmp_string_op(layerver, recver, op)
1277 except bb.utils.VersionStringException as vse:
1278 bb.fatal('Error parsing LAYERRECOMMENDS_%s: %s' % (c, str(vse)))
1279 if not res:
1280 parselog.debug(3,"Layer '%s' recommends version %s of layer '%s', but version %s is currently enabled in your configuration. Check that you are using the correct matching versions/branches of these two layers.", c, opstr, rec, layerver)
1281 continue
1282 else:
1283 parselog.debug(3,"Layer '%s' recommends version %s of layer '%s', which exists in your configuration but does not specify a version. Check that you are using the correct matching versions/branches of these two layers.", c, opstr, rec)
1284 continue
1285 parselog.debug(3,"Layer '%s' recommends layer '%s', so we are adding it", c, rec)
1286 collection_depends[c].append(rec)
1287 else:
1288 parselog.debug(3,"Layer '%s' recommends layer '%s', but this layer is not enabled in your configuration", c, rec)
1289
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001290 # Recursively work out collection priorities based on dependencies
1291 def calc_layer_priority(collection):
1292 if not collection_priorities[collection]:
1293 max_depprio = min_prio
1294 for dep in collection_depends[collection]:
1295 calc_layer_priority(dep)
1296 depprio = collection_priorities[dep]
1297 if depprio > max_depprio:
1298 max_depprio = depprio
1299 max_depprio += 1
1300 parselog.debug(1, "Calculated priority of layer %s as %d", collection, max_depprio)
1301 collection_priorities[collection] = max_depprio
1302
1303 # Calculate all layer priorities using calc_layer_priority and store in bbfile_config_priorities
1304 for c in collection_list:
1305 calc_layer_priority(c)
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001306 regex = self.data.getVar("BBFILE_PATTERN_%s" % c)
Andrew Geissler82c905d2020-04-13 13:39:40 -05001307 if regex is None:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001308 parselog.error("BBFILE_PATTERN_%s not defined" % c)
1309 errors = True
1310 continue
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001311 elif regex == "":
1312 parselog.debug(1, "BBFILE_PATTERN_%s is empty" % c)
Brad Bishop19323692019-04-05 15:28:33 -04001313 cre = re.compile('^NULL$')
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001314 errors = False
1315 else:
1316 try:
1317 cre = re.compile(regex)
1318 except re.error:
1319 parselog.error("BBFILE_PATTERN_%s \"%s\" is not a valid regular expression", c, regex)
1320 errors = True
1321 continue
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001322 self.bbfile_config_priorities.append((c, regex, cre, collection_priorities[c]))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001323 if errors:
1324 # We've already printed the actual error(s)
1325 raise CollectionError("Errors during parsing layer configuration")
1326
1327 def buildSetVars(self):
1328 """
1329 Setup any variables needed before starting a build
1330 """
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001331 t = time.gmtime()
1332 for mc in self.databuilder.mcdata:
1333 ds = self.databuilder.mcdata[mc]
1334 if not ds.getVar("BUILDNAME", False):
1335 ds.setVar("BUILDNAME", "${DATE}${TIME}")
1336 ds.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S', t))
1337 ds.setVar("DATE", time.strftime('%Y%m%d', t))
1338 ds.setVar("TIME", time.strftime('%H%M%S', t))
1339
1340 def reset_mtime_caches(self):
1341 """
1342 Reset mtime caches - this is particularly important when memory resident as something
1343 which is cached is not unlikely to have changed since the last invocation (e.g. a
1344 file associated with a recipe might have been modified by the user).
1345 """
1346 build.reset_cache()
1347 bb.fetch._checksum_cache.mtime_cache.clear()
1348 siggen_cache = getattr(bb.parse.siggen, 'checksum_cache', None)
1349 if siggen_cache:
1350 bb.parse.siggen.checksum_cache.mtime_cache.clear()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001351
Andrew Geissler5a43b432020-06-13 10:46:56 -05001352 def matchFiles(self, bf, mc=''):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001353 """
1354 Find the .bb files which match the expression in 'buildfile'.
1355 """
1356 if bf.startswith("/") or bf.startswith("../"):
1357 bf = os.path.abspath(bf)
1358
Andrew Geissler5a43b432020-06-13 10:46:56 -05001359 self.collections = {mc: CookerCollectFiles(self.bbfile_config_priorities, mc)}
1360 filelist, masked, searchdirs = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc])
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001361 try:
1362 os.stat(bf)
1363 bf = os.path.abspath(bf)
1364 return [bf]
1365 except OSError:
1366 regexp = re.compile(bf)
1367 matches = []
1368 for f in filelist:
1369 if regexp.search(f) and os.path.isfile(f):
1370 matches.append(f)
1371 return matches
1372
Andrew Geissler5a43b432020-06-13 10:46:56 -05001373 def matchFile(self, buildfile, mc=''):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001374 """
1375 Find the .bb file which matches the expression in 'buildfile'.
1376 Raise an error if multiple files
1377 """
Andrew Geissler5a43b432020-06-13 10:46:56 -05001378 matches = self.matchFiles(buildfile, mc)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001379 if len(matches) != 1:
1380 if matches:
1381 msg = "Unable to match '%s' to a specific recipe file - %s matches found:" % (buildfile, len(matches))
1382 if matches:
1383 for f in matches:
1384 msg += "\n %s" % f
1385 parselog.error(msg)
1386 else:
1387 parselog.error("Unable to find any recipe file matching '%s'" % buildfile)
1388 raise NoSpecificMatch
1389 return matches[0]
1390
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001391 def buildFile(self, buildfile, task):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001392 """
1393 Build the file matching regexp buildfile
1394 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001395 bb.event.fire(bb.event.BuildInit(), self.data)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001396
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001397 # Too many people use -b because they think it's how you normally
1398 # specify a target to be built, so show a warning
1399 bb.warn("Buildfile specified, dependencies will not be handled. If this is not what you want, do not use -b / --buildfile.")
1400
1401 self.buildFileInternal(buildfile, task)
1402
1403 def buildFileInternal(self, buildfile, task, fireevents=True, quietlog=False):
1404 """
1405 Build the file matching regexp buildfile
1406 """
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001407
1408 # Parse the configuration here. We need to do it explicitly here since
1409 # buildFile() doesn't use the cache
1410 self.parseConfiguration()
1411
1412 # If we are told to do the None task then query the default task
Andrew Geissler82c905d2020-04-13 13:39:40 -05001413 if task is None:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001414 task = self.configuration.cmd
Brad Bishop37a0e4d2017-12-04 01:01:44 -05001415 if not task.startswith("do_"):
1416 task = "do_%s" % task
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001417
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001418 fn, cls, mc = bb.cache.virtualfn2realfn(buildfile)
Andrew Geissler5a43b432020-06-13 10:46:56 -05001419 fn = self.matchFile(fn, mc)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001420
1421 self.buildSetVars()
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001422 self.reset_mtime_caches()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001423
Andrew Geissler5a43b432020-06-13 10:46:56 -05001424 bb_caches = bb.cache.MulticonfigCache(self.databuilder, self.data_hash, self.caches_array)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001425
Andrew Geissler5a43b432020-06-13 10:46:56 -05001426 infos = bb_caches[mc].parse(fn, self.collections[mc].get_file_appends(fn))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001427 infos = dict(infos)
1428
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001429 fn = bb.cache.realfn2virtual(fn, cls, mc)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001430 try:
1431 info_array = infos[fn]
1432 except KeyError:
1433 bb.fatal("%s does not exist" % fn)
1434
1435 if info_array[0].skipped:
1436 bb.fatal("%s was skipped: %s" % (fn, info_array[0].skipreason))
1437
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001438 self.recipecaches[mc].add_from_recipeinfo(fn, info_array)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001439
1440 # Tweak some variables
1441 item = info_array[0].pn
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001442 self.recipecaches[mc].ignored_dependencies = set()
1443 self.recipecaches[mc].bbfile_priority[fn] = 1
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001444 self.configuration.limited_deps = True
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001445
1446 # Remove external dependencies
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001447 self.recipecaches[mc].task_deps[fn]['depends'] = {}
1448 self.recipecaches[mc].deps[fn] = []
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001449 self.recipecaches[mc].rundeps[fn] = defaultdict(list)
1450 self.recipecaches[mc].runrecs[fn] = defaultdict(list)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001451
1452 # Invalidate task for target if force mode active
1453 if self.configuration.force:
1454 logger.verbose("Invalidate task %s, %s", task, fn)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001455 bb.parse.siggen.invalidate_task(task, self.recipecaches[mc], fn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001456
1457 # Setup taskdata structure
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001458 taskdata = {}
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001459 taskdata[mc] = bb.taskdata.TaskData(self.configuration.halt)
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001460 taskdata[mc].add_provider(self.databuilder.mcdata[mc], self.recipecaches[mc], item)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001461
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001462 if quietlog:
1463 rqloglevel = bb.runqueue.logger.getEffectiveLevel()
1464 bb.runqueue.logger.setLevel(logging.WARNING)
1465
1466 buildname = self.databuilder.mcdata[mc].getVar("BUILDNAME")
1467 if fireevents:
1468 bb.event.fire(bb.event.BuildStarted(buildname, [item]), self.databuilder.mcdata[mc])
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001469
1470 # Execute the runqueue
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001471 runlist = [[mc, item, task, fn]]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001472
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001473 rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001474
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001475 def buildFileIdle(server, rq, halt):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001476
1477 msg = None
1478 interrupted = 0
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001479 if halt or self.state == state.forceshutdown:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001480 rq.finish_runqueue(True)
1481 msg = "Forced shutdown"
1482 interrupted = 2
1483 elif self.state == state.shutdown:
1484 rq.finish_runqueue(False)
1485 msg = "Stopped build"
1486 interrupted = 1
1487 failures = 0
1488 try:
1489 retval = rq.execute_runqueue()
1490 except runqueue.TaskFailure as exc:
1491 failures += len(exc.args)
1492 retval = False
1493 except SystemExit as exc:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001494 self.command.finishAsyncCommand(str(exc))
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001495 if quietlog:
1496 bb.runqueue.logger.setLevel(rqloglevel)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001497 return False
1498
1499 if not retval:
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001500 if fireevents:
1501 bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runtaskentries), buildname, item, failures, interrupted), self.databuilder.mcdata[mc])
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001502 self.command.finishAsyncCommand(msg)
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001503 # We trashed self.recipecaches above
1504 self.parsecache_valid = False
1505 self.configuration.limited_deps = False
1506 bb.parse.siggen.reset(self.data)
1507 if quietlog:
1508 bb.runqueue.logger.setLevel(rqloglevel)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001509 return False
1510 if retval is True:
1511 return True
1512 return retval
1513
Andrew Geissler635e0e42020-08-21 15:58:33 -05001514 self.idleCallBackRegister(buildFileIdle, rq)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001515
1516 def buildTargets(self, targets, task):
1517 """
1518 Attempt to build the targets specified
1519 """
1520
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001521 def buildTargetsIdle(server, rq, halt):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001522 msg = None
1523 interrupted = 0
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001524 if halt or self.state == state.forceshutdown:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001525 rq.finish_runqueue(True)
1526 msg = "Forced shutdown"
1527 interrupted = 2
1528 elif self.state == state.shutdown:
1529 rq.finish_runqueue(False)
1530 msg = "Stopped build"
1531 interrupted = 1
1532 failures = 0
1533 try:
1534 retval = rq.execute_runqueue()
1535 except runqueue.TaskFailure as exc:
1536 failures += len(exc.args)
1537 retval = False
1538 except SystemExit as exc:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001539 self.command.finishAsyncCommand(str(exc))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001540 return False
1541
1542 if not retval:
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001543 try:
1544 for mc in self.multiconfigs:
1545 bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runtaskentries), buildname, targets, failures, interrupted), self.databuilder.mcdata[mc])
1546 finally:
1547 self.command.finishAsyncCommand(msg)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001548 return False
1549 if retval is True:
1550 return True
1551 return retval
1552
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001553 self.reset_mtime_caches()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001554 self.buildSetVars()
1555
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001556 # If we are told to do the None task then query the default task
Andrew Geissler82c905d2020-04-13 13:39:40 -05001557 if task is None:
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001558 task = self.configuration.cmd
1559
1560 if not task.startswith("do_"):
1561 task = "do_%s" % task
1562
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001563 packages = [target if ':' in target else '%s:%s' % (target, task) for target in targets]
1564
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001565 bb.event.fire(bb.event.BuildInit(packages), self.data)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001566
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001567 taskdata, runlist = self.buildTaskData(targets, task, self.configuration.halt)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001568
1569 buildname = self.data.getVar("BUILDNAME", False)
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001570
1571 # make targets to always look as <target>:do_<task>
1572 ntargets = []
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001573 for target in runlist:
1574 if target[0]:
Brad Bishop15ae2502019-06-18 21:44:24 -04001575 ntargets.append("mc:%s:%s:%s" % (target[0], target[1], target[2]))
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001576 ntargets.append("%s:%s" % (target[1], target[2]))
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001577
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001578 for mc in self.multiconfigs:
1579 bb.event.fire(bb.event.BuildStarted(buildname, ntargets), self.databuilder.mcdata[mc])
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001580
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001581 rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001582 if 'universe' in targets:
1583 rq.rqdata.warn_multi_bb = True
1584
Andrew Geissler635e0e42020-08-21 15:58:33 -05001585 self.idleCallBackRegister(buildTargetsIdle, rq)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001586
1587
1588 def getAllKeysWithFlags(self, flaglist):
1589 dump = {}
1590 for k in self.data.keys():
1591 try:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001592 expand = True
1593 flags = self.data.getVarFlags(k)
1594 if flags and "func" in flags and "python" in flags:
1595 expand = False
1596 v = self.data.getVar(k, expand)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001597 if not k.startswith("__") and not isinstance(v, bb.data_smart.DataSmart):
1598 dump[k] = {
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001599 'v' : str(v) ,
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001600 'history' : self.data.varhistory.variable(k),
1601 }
1602 for d in flaglist:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001603 if flags and d in flags:
1604 dump[k][d] = flags[d]
1605 else:
1606 dump[k][d] = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001607 except Exception as e:
1608 print(e)
1609 return dump
1610
1611
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001612 def updateCacheSync(self):
1613 if self.state == state.running:
1614 return
1615
1616 # reload files for which we got notifications
1617 for p in self.inotify_modified_files:
1618 bb.parse.update_cache(p)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001619 if p in bb.parse.BBHandler.cached_statements:
1620 del bb.parse.BBHandler.cached_statements[p]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001621 self.inotify_modified_files = []
1622
1623 if not self.baseconfig_valid:
Andrew Geisslerd1e89492021-02-12 15:35:20 -06001624 logger.debug("Reloading base configuration data")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001625 self.initConfigurationData()
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001626 self.handlePRServ()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001627
1628 # This is called for all async commands when self.state != running
1629 def updateCache(self):
1630 if self.state == state.running:
1631 return
1632
1633 if self.state in (state.shutdown, state.forceshutdown, state.error):
1634 if hasattr(self.parser, 'shutdown'):
Andrew Geissler9aee5002022-03-30 16:27:02 +00001635 self.parser.shutdown(clean=False)
Andrew Geisslerc9f78652020-09-18 14:11:35 -05001636 self.parser.final_cleanup()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001637 raise bb.BBHandledException()
1638
1639 if self.state != state.parsing:
1640 self.updateCacheSync()
1641
1642 if self.state != state.parsing and not self.parsecache_valid:
Patrick Williamsde0582f2022-04-08 10:23:27 -05001643 self.setupParserWatcher()
1644
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001645 bb.parse.siggen.reset(self.data)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001646 self.parseConfiguration ()
1647 if CookerFeatures.SEND_SANITYEVENTS in self.featureset:
Brad Bishop37a0e4d2017-12-04 01:01:44 -05001648 for mc in self.multiconfigs:
1649 bb.event.fire(bb.event.SanityCheck(False), self.databuilder.mcdata[mc])
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001650
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001651 for mc in self.multiconfigs:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001652 ignore = self.databuilder.mcdata[mc].getVar("ASSUME_PROVIDED") or ""
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001653 self.recipecaches[mc].ignored_dependencies = set(ignore.split())
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001654
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001655 for dep in self.configuration.extra_assume_provided:
1656 self.recipecaches[mc].ignored_dependencies.add(dep)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001657
Andrew Geissler5a43b432020-06-13 10:46:56 -05001658 self.collections = {}
1659
1660 mcfilelist = {}
1661 total_masked = 0
1662 searchdirs = set()
1663 for mc in self.multiconfigs:
1664 self.collections[mc] = CookerCollectFiles(self.bbfile_config_priorities, mc)
1665 (filelist, masked, search) = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc])
1666
1667 mcfilelist[mc] = filelist
1668 total_masked += masked
1669 searchdirs |= set(search)
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001670
1671 # Add inotify watches for directories searched for bb/bbappend files
1672 for dirent in searchdirs:
1673 self.add_filewatch([[dirent]], dirs=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001674
Andrew Geissler5a43b432020-06-13 10:46:56 -05001675 self.parser = CookerParser(self, mcfilelist, total_masked)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001676 self.parsecache_valid = True
1677
1678 self.state = state.parsing
1679
1680 if not self.parser.parse_next():
1681 collectlog.debug(1, "parsing complete")
1682 if self.parser.error:
1683 raise bb.BBHandledException()
1684 self.show_appends_with_no_recipes()
1685 self.handlePrefProviders()
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001686 for mc in self.multiconfigs:
Andrew Geisslerb7d28612020-07-24 16:15:54 -05001687 self.recipecaches[mc].bbfile_priority = self.collections[mc].collection_priorities(self.recipecaches[mc].pkg_fn, self.parser.mcfilelist[mc], self.data)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001688 self.state = state.running
1689
1690 # Send an event listing all stamps reachable after parsing
1691 # which the metadata may use to clean up stale data
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001692 for mc in self.multiconfigs:
1693 event = bb.event.ReachableStamps(self.recipecaches[mc].stamp)
1694 bb.event.fire(event, self.databuilder.mcdata[mc])
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001695 return None
1696
1697 return True
1698
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001699 def checkPackages(self, pkgs_to_build, task=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001700
1701 # Return a copy, don't modify the original
1702 pkgs_to_build = pkgs_to_build[:]
1703
Andrew Geissler595f6302022-01-24 19:11:47 +00001704 if not pkgs_to_build:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001705 raise NothingToBuild
1706
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001707 ignore = (self.data.getVar("ASSUME_PROVIDED") or "").split()
Andrew Geisslerb7d28612020-07-24 16:15:54 -05001708 for pkg in pkgs_to_build.copy():
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001709 if pkg in ignore:
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001710 parselog.warning("Explicit target \"%s\" is in ASSUME_PROVIDED, ignoring" % pkg)
Brad Bishop15ae2502019-06-18 21:44:24 -04001711 if pkg.startswith("multiconfig:"):
1712 pkgs_to_build.remove(pkg)
1713 pkgs_to_build.append(pkg.replace("multiconfig:", "mc:"))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001714
1715 if 'world' in pkgs_to_build:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001716 pkgs_to_build.remove('world')
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001717 for mc in self.multiconfigs:
1718 bb.providers.buildWorldTargetList(self.recipecaches[mc], task)
1719 for t in self.recipecaches[mc].world_target:
1720 if mc:
Brad Bishop15ae2502019-06-18 21:44:24 -04001721 t = "mc:" + mc + ":" + t
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001722 pkgs_to_build.append(t)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001723
1724 if 'universe' in pkgs_to_build:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001725 parselog.verbnote("The \"universe\" target is only intended for testing and may produce errors.")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001726 parselog.debug(1, "collating packages for \"universe\"")
1727 pkgs_to_build.remove('universe')
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001728 for mc in self.multiconfigs:
1729 for t in self.recipecaches[mc].universe_target:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001730 if task:
1731 foundtask = False
1732 for provider_fn in self.recipecaches[mc].providers[t]:
1733 if task in self.recipecaches[mc].task_deps[provider_fn]['tasks']:
1734 foundtask = True
1735 break
1736 if not foundtask:
1737 bb.debug(1, "Skipping %s for universe tasks as task %s doesn't exist" % (t, task))
1738 continue
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001739 if mc:
Brad Bishop15ae2502019-06-18 21:44:24 -04001740 t = "mc:" + mc + ":" + t
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001741 pkgs_to_build.append(t)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001742
1743 return pkgs_to_build
1744
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001745 def pre_serve(self):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001746 return
1747
1748 def post_serve(self):
Andrew Geisslerc9f78652020-09-18 14:11:35 -05001749 self.shutdown(force=True)
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001750 prserv.serv.auto_shutdown()
Patrick Williams45852732022-04-02 08:58:32 -05001751 if hasattr(bb.parse, "siggen"):
1752 bb.parse.siggen.exit()
Brad Bishop08902b02019-08-20 09:16:51 -04001753 if self.hashserv:
1754 self.hashserv.process.terminate()
1755 self.hashserv.process.join()
Andrew Geisslerc3d88e42020-10-02 09:45:00 -05001756 if hasattr(self, "data"):
1757 bb.event.fire(CookerExit(), self.data)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001758
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001759 def shutdown(self, force = False):
1760 if force:
1761 self.state = state.forceshutdown
1762 else:
1763 self.state = state.shutdown
1764
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001765 if self.parser:
Andrew Geissler9aee5002022-03-30 16:27:02 +00001766 self.parser.shutdown(clean=not force)
Andrew Geisslerc9f78652020-09-18 14:11:35 -05001767 self.parser.final_cleanup()
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001768
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001769 def finishcommand(self):
1770 self.state = state.initial
1771
1772 def reset(self):
Patrick Williams45852732022-04-02 08:58:32 -05001773 if hasattr(bb.parse, "siggen"):
1774 bb.parse.siggen.exit()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001775 self.initConfigurationData()
Brad Bishop08902b02019-08-20 09:16:51 -04001776 self.handlePRServ()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001777
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001778 def clientComplete(self):
1779 """Called when the client is done using the server"""
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001780 self.finishcommand()
1781 self.extraconfigdata = {}
1782 self.command.reset()
Andrew Geisslerc9f78652020-09-18 14:11:35 -05001783 if hasattr(self, "data"):
1784 self.databuilder.reset()
1785 self.data = self.databuilder.data
Andrew Geissler82c905d2020-04-13 13:39:40 -05001786 self.parsecache_valid = False
1787 self.baseconfig_valid = False
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001788
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001789
1790class CookerExit(bb.event.Event):
1791 """
1792 Notify clients of the Cooker shutdown
1793 """
1794
1795 def __init__(self):
1796 bb.event.Event.__init__(self)
1797
1798
1799class CookerCollectFiles(object):
Andrew Geissler5a43b432020-06-13 10:46:56 -05001800 def __init__(self, priorities, mc=''):
1801 self.mc = mc
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001802 self.bbappends = []
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001803 # Priorities is a list of tuples, with the second element as the pattern.
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001804 # We need to sort the list with the longest pattern first, and so on to
1805 # the shortest. This allows nested layers to be properly evaluated.
1806 self.bbfile_config_priorities = sorted(priorities, key=lambda tup: tup[1], reverse=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001807
Andrew Geisslerb7d28612020-07-24 16:15:54 -05001808 def calc_bbfile_priority(self, filename):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001809 for _, _, regex, pri in self.bbfile_config_priorities:
1810 if regex.match(filename):
Andrew Geisslerb7d28612020-07-24 16:15:54 -05001811 return pri, regex
1812 return 0, None
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001813
1814 def get_bbfiles(self):
1815 """Get list of default .bb files by reading out the current directory"""
1816 path = os.getcwd()
1817 contents = os.listdir(path)
1818 bbfiles = []
1819 for f in contents:
1820 if f.endswith(".bb"):
1821 bbfiles.append(os.path.abspath(os.path.join(path, f)))
1822 return bbfiles
1823
1824 def find_bbfiles(self, path):
1825 """Find all the .bb and .bbappend files in a directory"""
1826 found = []
1827 for dir, dirs, files in os.walk(path):
1828 for ignored in ('SCCS', 'CVS', '.svn'):
1829 if ignored in dirs:
1830 dirs.remove(ignored)
1831 found += [os.path.join(dir, f) for f in files if (f.endswith(['.bb', '.bbappend']))]
1832
1833 return found
1834
1835 def collect_bbfiles(self, config, eventdata):
1836 """Collect all available .bb build files"""
1837 masked = 0
1838
1839 collectlog.debug(1, "collecting .bb files")
1840
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001841 files = (config.getVar( "BBFILES") or "").split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001842
1843 # Sort files by priority
Andrew Geisslerb7d28612020-07-24 16:15:54 -05001844 files.sort( key=lambda fileitem: self.calc_bbfile_priority(fileitem)[0] )
Andrew Geisslerc9f78652020-09-18 14:11:35 -05001845 config.setVar("BBFILES_PRIORITIZED", " ".join(files))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001846
Andrew Geissler595f6302022-01-24 19:11:47 +00001847 if not files:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001848 files = self.get_bbfiles()
1849
Andrew Geissler595f6302022-01-24 19:11:47 +00001850 if not files:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001851 collectlog.error("no recipe files to build, check your BBPATH and BBFILES?")
1852 bb.event.fire(CookerExit(), eventdata)
1853
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001854 # We need to track where we look so that we can add inotify watches. There
1855 # is no nice way to do this, this is horrid. We intercept the os.listdir()
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001856 # (or os.scandir() for python 3.6+) calls while we run glob().
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001857 origlistdir = os.listdir
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001858 if hasattr(os, 'scandir'):
1859 origscandir = os.scandir
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001860 searchdirs = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001861
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001862 def ourlistdir(d):
1863 searchdirs.append(d)
1864 return origlistdir(d)
1865
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001866 def ourscandir(d):
1867 searchdirs.append(d)
1868 return origscandir(d)
1869
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001870 os.listdir = ourlistdir
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001871 if hasattr(os, 'scandir'):
1872 os.scandir = ourscandir
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001873 try:
1874 # Can't use set here as order is important
1875 newfiles = []
1876 for f in files:
1877 if os.path.isdir(f):
1878 dirfiles = self.find_bbfiles(f)
1879 for g in dirfiles:
1880 if g not in newfiles:
1881 newfiles.append(g)
1882 else:
1883 globbed = glob.glob(f)
1884 if not globbed and os.path.exists(f):
1885 globbed = [f]
1886 # glob gives files in order on disk. Sort to be deterministic.
1887 for g in sorted(globbed):
1888 if g not in newfiles:
1889 newfiles.append(g)
1890 finally:
1891 os.listdir = origlistdir
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001892 if hasattr(os, 'scandir'):
1893 os.scandir = origscandir
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001894
1895 bbmask = config.getVar('BBMASK')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001896
1897 if bbmask:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001898 # First validate the individual regular expressions and ignore any
1899 # that do not compile
1900 bbmasks = []
1901 for mask in bbmask.split():
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001902 # When constructing an older style single regex, it's possible for BBMASK
1903 # to end up beginning with '|', which matches and masks _everything_.
1904 if mask.startswith("|"):
Andrew Geissler82c905d2020-04-13 13:39:40 -05001905 collectlog.warning("BBMASK contains regular expression beginning with '|', fixing: %s" % mask)
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001906 mask = mask[1:]
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001907 try:
1908 re.compile(mask)
1909 bbmasks.append(mask)
Andrew Geissler78b72792022-06-14 06:47:25 -05001910 except re.error:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001911 collectlog.critical("BBMASK contains an invalid regular expression, ignoring: %s" % mask)
1912
1913 # Then validate the combined regular expressions. This should never
1914 # fail, but better safe than sorry...
1915 bbmask = "|".join(bbmasks)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001916 try:
1917 bbmask_compiled = re.compile(bbmask)
Andrew Geissler78b72792022-06-14 06:47:25 -05001918 except re.error:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001919 collectlog.critical("BBMASK is not a valid regular expression, ignoring: %s" % bbmask)
1920 bbmask = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001921
1922 bbfiles = []
1923 bbappend = []
1924 for f in newfiles:
1925 if bbmask and bbmask_compiled.search(f):
1926 collectlog.debug(1, "skipping masked file %s", f)
1927 masked += 1
1928 continue
1929 if f.endswith('.bb'):
1930 bbfiles.append(f)
1931 elif f.endswith('.bbappend'):
1932 bbappend.append(f)
1933 else:
1934 collectlog.debug(1, "skipping %s: unknown file extension", f)
1935
1936 # Build a list of .bbappend files for each .bb file
1937 for f in bbappend:
1938 base = os.path.basename(f).replace('.bbappend', '.bb')
1939 self.bbappends.append((base, f))
1940
1941 # Find overlayed recipes
1942 # bbfiles will be in priority order which makes this easy
1943 bbfile_seen = dict()
1944 self.overlayed = defaultdict(list)
1945 for f in reversed(bbfiles):
1946 base = os.path.basename(f)
1947 if base not in bbfile_seen:
1948 bbfile_seen[base] = f
1949 else:
1950 topfile = bbfile_seen[base]
1951 self.overlayed[topfile].append(f)
1952
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001953 return (bbfiles, masked, searchdirs)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001954
1955 def get_file_appends(self, fn):
1956 """
1957 Returns a list of .bbappend files to apply to fn
1958 """
1959 filelist = []
1960 f = os.path.basename(fn)
1961 for b in self.bbappends:
1962 (bbappend, filename) = b
1963 if (bbappend == f) or ('%' in bbappend and bbappend.startswith(f[:bbappend.index('%')])):
1964 filelist.append(filename)
Andrew Geissler5a43b432020-06-13 10:46:56 -05001965 return tuple(filelist)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001966
Andrew Geisslerb7d28612020-07-24 16:15:54 -05001967 def collection_priorities(self, pkgfns, fns, d):
1968 # Return the priorities of the entries in pkgfns
1969 # Also check that all the regexes in self.bbfile_config_priorities are used
1970 # (but to do that we need to ensure skipped recipes aren't counted, nor
1971 # collections in BBFILE_PATTERN_IGNORE_EMPTY)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001972
1973 priorities = {}
Andrew Geisslerb7d28612020-07-24 16:15:54 -05001974 seen = set()
1975 matched = set()
1976
1977 matched_regex = set()
1978 unmatched_regex = set()
1979 for _, _, regex, _ in self.bbfile_config_priorities:
1980 unmatched_regex.add(regex)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001981
1982 # Calculate priorities for each file
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001983 for p in pkgfns:
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001984 realfn, cls, mc = bb.cache.virtualfn2realfn(p)
Andrew Geisslerb7d28612020-07-24 16:15:54 -05001985 priorities[p], regex = self.calc_bbfile_priority(realfn)
1986 if regex in unmatched_regex:
1987 matched_regex.add(regex)
1988 unmatched_regex.remove(regex)
1989 seen.add(realfn)
1990 if regex:
1991 matched.add(realfn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001992
Andrew Geisslerb7d28612020-07-24 16:15:54 -05001993 if unmatched_regex:
1994 # Account for bbappend files
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001995 for b in self.bbappends:
1996 (bbfile, append) = b
Andrew Geisslerb7d28612020-07-24 16:15:54 -05001997 seen.add(append)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001998
Andrew Geisslerb7d28612020-07-24 16:15:54 -05001999 # Account for skipped recipes
2000 seen.update(fns)
2001
2002 seen.difference_update(matched)
2003
2004 def already_matched(fn):
2005 for regex in matched_regex:
2006 if regex.match(fn):
2007 return True
2008 return False
2009
2010 for unmatch in unmatched_regex.copy():
2011 for fn in seen:
2012 if unmatch.match(fn):
2013 # If the bbappend or file was already matched by another regex, skip it
2014 # e.g. for a layer within a layer, the outer regex could match, the inner
2015 # regex may match nothing and we should warn about that
2016 if already_matched(fn):
2017 continue
2018 unmatched_regex.remove(unmatch)
2019 break
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002020
2021 for collection, pattern, regex, _ in self.bbfile_config_priorities:
Andrew Geisslerb7d28612020-07-24 16:15:54 -05002022 if regex in unmatched_regex:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05002023 if d.getVar('BBFILE_PATTERN_IGNORE_EMPTY_%s' % collection) != '1':
Andrew Geissler5a43b432020-06-13 10:46:56 -05002024 collectlog.warning("No bb files in %s matched BBFILE_PATTERN_%s '%s'" % (self.mc if self.mc else 'default',
2025 collection, pattern))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002026
2027 return priorities
2028
2029class ParsingFailure(Exception):
2030 def __init__(self, realexception, recipe):
2031 self.realexception = realexception
2032 self.recipe = recipe
2033 Exception.__init__(self, realexception, recipe)
2034
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002035class Parser(multiprocessing.Process):
Andrew Geissler9aee5002022-03-30 16:27:02 +00002036 def __init__(self, jobs, results, quit, profile):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002037 self.jobs = jobs
2038 self.results = results
2039 self.quit = quit
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002040 multiprocessing.Process.__init__(self)
2041 self.context = bb.utils.get_context().copy()
2042 self.handlers = bb.event.get_class_handlers().copy()
2043 self.profile = profile
Andrew Geissler9aee5002022-03-30 16:27:02 +00002044 self.queue_signals = False
2045 self.signal_received = []
2046 self.signal_threadlock = threading.Lock()
2047
2048 def catch_sig(self, signum, frame):
2049 if self.queue_signals:
2050 self.signal_received.append(signum)
2051 else:
2052 self.handle_sig(signum, frame)
2053
2054 def handle_sig(self, signum, frame):
2055 if signum == signal.SIGTERM:
2056 signal.signal(signal.SIGTERM, signal.SIG_DFL)
2057 os.kill(os.getpid(), signal.SIGTERM)
2058 elif signum == signal.SIGINT:
2059 signal.default_int_handler(signum, frame)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002060
2061 def run(self):
2062
2063 if not self.profile:
2064 self.realrun()
2065 return
2066
2067 try:
2068 import cProfile as profile
2069 except:
2070 import profile
2071 prof = profile.Profile()
2072 try:
2073 profile.Profile.runcall(prof, self.realrun)
2074 finally:
2075 logfile = "profile-parse-%s.log" % multiprocessing.current_process().name
2076 prof.dump_stats(logfile)
2077
2078 def realrun(self):
Andrew Geissler9aee5002022-03-30 16:27:02 +00002079 # Signal handling here is hard. We must not terminate any process or thread holding the write
2080 # lock for the event stream as it will not be released, ever, and things will hang.
2081 # Python handles signals in the main thread/process but they can be raised from any thread and
2082 # we want to defer processing of any SIGTERM/SIGINT signal until we're outside the critical section
2083 # and don't hold the lock (see server/process.py). We therefore always catch the signals (so any
2084 # new thread should also do so) and we defer handling but we handle with the local thread lock
2085 # held (a threading lock, not a multiprocessing one) so that no other thread in the process
2086 # can be in the critical section.
2087 signal.signal(signal.SIGTERM, self.catch_sig)
2088 signal.signal(signal.SIGHUP, signal.SIG_DFL)
2089 signal.signal(signal.SIGINT, self.catch_sig)
2090 bb.utils.set_process_name(multiprocessing.current_process().name)
2091 multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, exitpriority=1)
2092 multiprocessing.util.Finalize(None, bb.fetch.fetcher_parse_save, exitpriority=1)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002093
2094 pending = []
Andrew Geissler9aee5002022-03-30 16:27:02 +00002095 try:
2096 while True:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002097 try:
Andrew Geissler9aee5002022-03-30 16:27:02 +00002098 self.quit.get_nowait()
2099 except queue.Empty:
2100 pass
2101 else:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002102 break
Andrew Geissler9aee5002022-03-30 16:27:02 +00002103
2104 if pending:
2105 result = pending.pop()
2106 else:
2107 try:
2108 job = self.jobs.pop()
2109 except IndexError:
2110 break
2111 result = self.parse(*job)
2112 # Clear the siggen cache after parsing to control memory usage, its huge
2113 bb.parse.siggen.postparsing_clean_cache()
2114 try:
2115 self.results.put(result, timeout=0.25)
2116 except queue.Full:
2117 pending.append(result)
2118 finally:
2119 self.results.close()
2120 self.results.join_thread()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002121
Andrew Geissler5a43b432020-06-13 10:46:56 -05002122 def parse(self, mc, cache, filename, appends):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002123 try:
Andrew Geissler82c905d2020-04-13 13:39:40 -05002124 origfilter = bb.event.LogHandler.filter
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05002125 # Record the filename we're parsing into any events generated
2126 def parse_filter(self, record):
2127 record.taskpid = bb.event.worker_pid
2128 record.fn = filename
2129 return True
2130
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002131 # Reset our environment and handlers to the original settings
2132 bb.utils.set_context(self.context.copy())
2133 bb.event.set_class_handlers(self.handlers.copy())
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05002134 bb.event.LogHandler.filter = parse_filter
2135
Andrew Geissler5a43b432020-06-13 10:46:56 -05002136 return True, mc, cache.parse(filename, appends)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002137 except Exception as exc:
2138 tb = sys.exc_info()[2]
2139 exc.recipe = filename
2140 exc.traceback = list(bb.exceptions.extract_traceback(tb, context=3))
Andrew Geissler9aee5002022-03-30 16:27:02 +00002141 return True, None, exc
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002142 # Need to turn BaseExceptions into Exceptions here so we gracefully shutdown
2143 # and for example a worker thread doesn't just exit on its own in response to
2144 # a SystemExit event for example.
2145 except BaseException as exc:
Andrew Geissler9aee5002022-03-30 16:27:02 +00002146 return True, None, ParsingFailure(exc, filename)
Andrew Geissler82c905d2020-04-13 13:39:40 -05002147 finally:
2148 bb.event.LogHandler.filter = origfilter
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002149
2150class CookerParser(object):
Andrew Geissler5a43b432020-06-13 10:46:56 -05002151 def __init__(self, cooker, mcfilelist, masked):
2152 self.mcfilelist = mcfilelist
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002153 self.cooker = cooker
2154 self.cfgdata = cooker.data
2155 self.cfghash = cooker.data_hash
Patrick Williamsc0f7c042017-02-23 20:41:17 -06002156 self.cfgbuilder = cooker.databuilder
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002157
2158 # Accounting statistics
2159 self.parsed = 0
2160 self.cached = 0
2161 self.error = 0
2162 self.masked = masked
2163
2164 self.skipped = 0
2165 self.virtuals = 0
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002166
2167 self.current = 0
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002168 self.process_names = []
2169
Andrew Geissler5a43b432020-06-13 10:46:56 -05002170 self.bb_caches = bb.cache.MulticonfigCache(self.cfgbuilder, self.cfghash, cooker.caches_array)
2171 self.fromcache = set()
2172 self.willparse = set()
2173 for mc in self.cooker.multiconfigs:
2174 for filename in self.mcfilelist[mc]:
2175 appends = self.cooker.collections[mc].get_file_appends(filename)
2176 if not self.bb_caches[mc].cacheValid(filename, appends):
2177 self.willparse.add((mc, self.bb_caches[mc], filename, appends))
2178 else:
2179 self.fromcache.add((mc, self.bb_caches[mc], filename, appends))
2180
2181 self.total = len(self.fromcache) + len(self.willparse)
2182 self.toparse = len(self.willparse)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06002183 self.progress_chunk = int(max(self.toparse / 100, 1))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002184
Brad Bishop6e60e8b2018-02-01 10:27:11 -05002185 self.num_processes = min(int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS") or
Andrew Geissler5a43b432020-06-13 10:46:56 -05002186 multiprocessing.cpu_count()), self.toparse)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05002187
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002188 self.start()
2189 self.haveshutdown = False
Andrew Geisslerc9f78652020-09-18 14:11:35 -05002190 self.syncthread = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002191
2192 def start(self):
2193 self.results = self.load_cached()
2194 self.processes = []
2195 if self.toparse:
2196 bb.event.fire(bb.event.ParseStarted(self.toparse), self.cfgdata)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002197
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002198 self.parser_quit = multiprocessing.Queue(maxsize=self.num_processes)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002199 self.result_queue = multiprocessing.Queue()
Brad Bishop19323692019-04-05 15:28:33 -04002200
2201 def chunkify(lst,n):
2202 return [lst[i::n] for i in range(n)]
Andrew Geissler5a43b432020-06-13 10:46:56 -05002203 self.jobs = chunkify(list(self.willparse), self.num_processes)
Brad Bishop19323692019-04-05 15:28:33 -04002204
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002205 for i in range(0, self.num_processes):
Andrew Geissler9aee5002022-03-30 16:27:02 +00002206 parser = Parser(self.jobs[i], self.result_queue, self.parser_quit, self.cooker.configuration.profile)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002207 parser.start()
2208 self.process_names.append(parser.name)
2209 self.processes.append(parser)
2210
2211 self.results = itertools.chain(self.results, self.parse_generator())
2212
Andrew Geissler9aee5002022-03-30 16:27:02 +00002213 def shutdown(self, clean=True):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002214 if not self.toparse:
2215 return
2216 if self.haveshutdown:
2217 return
2218 self.haveshutdown = True
2219
2220 if clean:
2221 event = bb.event.ParseCompleted(self.cached, self.parsed,
2222 self.skipped, self.masked,
2223 self.virtuals, self.error,
2224 self.total)
2225
2226 bb.event.fire(event, self.cfgdata)
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00002227 else:
Andrew Geissler9aee5002022-03-30 16:27:02 +00002228 bb.error("Parsing halted due to errors, see error messages above")
Andrew Geisslerc9f78652020-09-18 14:11:35 -05002229
2230 for process in self.processes:
2231 self.parser_quit.put(None)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002232
Brad Bishop08902b02019-08-20 09:16:51 -04002233 # Cleanup the queue before call process.join(), otherwise there might be
2234 # deadlocks.
2235 while True:
2236 try:
2237 self.result_queue.get(timeout=0.25)
2238 except queue.Empty:
2239 break
2240
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002241 for process in self.processes:
Andrew Geissler9aee5002022-03-30 16:27:02 +00002242 process.join(0.5)
2243
2244 for process in self.processes:
2245 if process.exitcode is None:
2246 os.kill(process.pid, signal.SIGINT)
2247
2248 for process in self.processes:
2249 process.join(0.5)
2250
2251 for process in self.processes:
2252 if process.exitcode is None:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002253 process.terminate()
Andrew Geissler9aee5002022-03-30 16:27:02 +00002254
2255 for process in self.processes:
2256 process.join()
2257 # Added in 3.7, cleans up zombies
2258 if hasattr(process, "close"):
2259 process.close()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002260
Andrew Geisslerc9f78652020-09-18 14:11:35 -05002261 self.parser_quit.close()
2262 # Allow data left in the cancel queue to be discarded
2263 self.parser_quit.cancel_join_thread()
2264
Andrew Geissler5a43b432020-06-13 10:46:56 -05002265 def sync_caches():
2266 for c in self.bb_caches.values():
2267 c.sync()
2268
Andrew Geisslerc9f78652020-09-18 14:11:35 -05002269 sync = threading.Thread(target=sync_caches, name="SyncThread")
2270 self.syncthread = sync
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002271 sync.start()
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05002272 bb.codeparser.parser_cache_savemerge()
2273 bb.fetch.fetcher_parse_done()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002274 if self.cooker.configuration.profile:
2275 profiles = []
2276 for i in self.process_names:
2277 logfile = "profile-parse-%s.log" % i
2278 if os.path.exists(logfile):
2279 profiles.append(logfile)
2280
2281 pout = "profile-parse.log.processed"
2282 bb.utils.process_profilelog(profiles, pout = pout)
2283 print("Processed parsing statistics saved to %s" % (pout))
2284
Andrew Geisslerc9f78652020-09-18 14:11:35 -05002285 def final_cleanup(self):
2286 if self.syncthread:
2287 self.syncthread.join()
2288
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002289 def load_cached(self):
Andrew Geissler5a43b432020-06-13 10:46:56 -05002290 for mc, cache, filename, appends in self.fromcache:
2291 cached, infos = cache.load(filename, appends)
2292 yield not cached, mc, infos
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002293
2294 def parse_generator(self):
Andrew Geissler595f6302022-01-24 19:11:47 +00002295 empty = False
2296 while self.processes or not empty:
2297 for process in self.processes.copy():
2298 if not process.is_alive():
2299 process.join()
2300 self.processes.remove(process)
2301
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002302 if self.parsed >= self.toparse:
2303 break
2304
2305 try:
2306 result = self.result_queue.get(timeout=0.25)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06002307 except queue.Empty:
Andrew Geissler595f6302022-01-24 19:11:47 +00002308 empty = True
Andrew Geissler9aee5002022-03-30 16:27:02 +00002309 yield None, None, None
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002310 else:
Andrew Geissler595f6302022-01-24 19:11:47 +00002311 empty = False
Andrew Geissler9aee5002022-03-30 16:27:02 +00002312 yield result
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002313
Andrew Geissler595f6302022-01-24 19:11:47 +00002314 if not (self.parsed >= self.toparse):
2315 raise bb.parse.ParseError("Not all recipes parsed, parser thread killed/died? Exiting.", None)
2316
2317
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002318 def parse_next(self):
2319 result = []
2320 parsed = None
2321 try:
Andrew Geissler5a43b432020-06-13 10:46:56 -05002322 parsed, mc, result = next(self.results)
Andrew Geissler9aee5002022-03-30 16:27:02 +00002323 if isinstance(result, BaseException):
2324 # Turn exceptions back into exceptions
2325 raise result
2326 if parsed is None:
2327 # Timeout, loop back through the main loop
2328 return True
2329
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002330 except StopIteration:
2331 self.shutdown()
2332 return False
2333 except bb.BBHandledException as exc:
2334 self.error += 1
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00002335 logger.debug('Failed to parse recipe: %s' % exc.recipe)
Andrew Geissler9aee5002022-03-30 16:27:02 +00002336 self.shutdown(clean=False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002337 return False
2338 except ParsingFailure as exc:
2339 self.error += 1
2340 logger.error('Unable to parse %s: %s' %
2341 (exc.recipe, bb.exceptions.to_string(exc.realexception)))
Andrew Geissler9aee5002022-03-30 16:27:02 +00002342 self.shutdown(clean=False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002343 return False
2344 except bb.parse.ParseError as exc:
2345 self.error += 1
2346 logger.error(str(exc))
Andrew Geissler9aee5002022-03-30 16:27:02 +00002347 self.shutdown(clean=False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002348 return False
2349 except bb.data_smart.ExpansionError as exc:
2350 self.error += 1
Patrick Williamsc0f7c042017-02-23 20:41:17 -06002351 bbdir = os.path.dirname(__file__) + os.sep
2352 etype, value, _ = sys.exc_info()
2353 tb = list(itertools.dropwhile(lambda e: e.filename.startswith(bbdir), exc.traceback))
2354 logger.error('ExpansionError during parsing %s', value.recipe,
2355 exc_info=(etype, value, tb))
Andrew Geissler9aee5002022-03-30 16:27:02 +00002356 self.shutdown(clean=False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002357 return False
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002358 except Exception as exc:
2359 self.error += 1
2360 etype, value, tb = sys.exc_info()
2361 if hasattr(value, "recipe"):
Patrick Williamsc0f7c042017-02-23 20:41:17 -06002362 logger.error('Unable to parse %s' % value.recipe,
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002363 exc_info=(etype, value, exc.traceback))
2364 else:
2365 # Most likely, an exception occurred during raising an exception
2366 import traceback
2367 logger.error('Exception during parse: %s' % traceback.format_exc())
Andrew Geissler9aee5002022-03-30 16:27:02 +00002368 self.shutdown(clean=False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002369 return False
2370
2371 self.current += 1
2372 self.virtuals += len(result)
2373 if parsed:
2374 self.parsed += 1
2375 if self.parsed % self.progress_chunk == 0:
2376 bb.event.fire(bb.event.ParseProgress(self.parsed, self.toparse),
2377 self.cfgdata)
2378 else:
2379 self.cached += 1
2380
2381 for virtualfn, info_array in result:
2382 if info_array[0].skipped:
2383 self.skipped += 1
2384 self.cooker.skiplist[virtualfn] = SkippedPackage(info_array[0])
Andrew Geissler5a43b432020-06-13 10:46:56 -05002385 self.bb_caches[mc].add_info(virtualfn, info_array, self.cooker.recipecaches[mc],
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002386 parsed=parsed, watcher = self.cooker.add_filewatch)
2387 return True
2388
2389 def reparse(self, filename):
Andrew Geissler5a43b432020-06-13 10:46:56 -05002390 to_reparse = set()
2391 for mc in self.cooker.multiconfigs:
2392 to_reparse.add((mc, filename, self.cooker.collections[mc].get_file_appends(filename)))
2393
2394 for mc, filename, appends in to_reparse:
2395 infos = self.bb_caches[mc].parse(filename, appends)
2396 for vfn, info_array in infos:
2397 self.cooker.recipecaches[mc].add_from_recipeinfo(vfn, info_array)