blob: cfaa7cbe6c2bf28c01a4472fb0fc852f56037d74 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001#
2# Copyright (C) 2003, 2004 Chris Larson
3# Copyright (C) 2003, 2004 Phil Blundell
4# Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
5# Copyright (C) 2005 Holger Hans Peter Freyther
6# Copyright (C) 2005 ROAD GmbH
7# Copyright (C) 2006 - 2007 Richard Purdie
8#
Brad Bishopc342db32019-05-15 21:57:59 -04009# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -050010#
Patrick Williamsc0f7c042017-02-23 20:41:17 -060011
Patrick Williamsc124f4f2015-09-15 14:41:29 -050012import sys, os, glob, os.path, re, time
Patrick Williamsc124f4f2015-09-15 14:41:29 -050013import itertools
14import logging
15import multiprocessing
Patrick Williamsc124f4f2015-09-15 14:41:29 -050016import threading
Patrick Williamsc0f7c042017-02-23 20:41:17 -060017from io import StringIO, UnsupportedOperation
Patrick Williamsc124f4f2015-09-15 14:41:29 -050018from contextlib import closing
Patrick Williamsc0f7c042017-02-23 20:41:17 -060019from collections import defaultdict, namedtuple
Patrick Williamsc124f4f2015-09-15 14:41:29 -050020import bb, bb.exceptions, bb.command
21from bb import utils, data, parse, event, cache, providers, taskdata, runqueue, build
Patrick Williamsc0f7c042017-02-23 20:41:17 -060022import queue
Patrick Williamsc124f4f2015-09-15 14:41:29 -050023import signal
Patrick Williamsc124f4f2015-09-15 14:41:29 -050024import prserv.serv
25import pyinotify
Patrick Williamsc0f7c042017-02-23 20:41:17 -060026import json
27import pickle
28import codecs
Brad Bishop08902b02019-08-20 09:16:51 -040029import hashserv
Patrick Williamsc124f4f2015-09-15 14:41:29 -050030
31logger = logging.getLogger("BitBake")
32collectlog = logging.getLogger("BitBake.Collection")
33buildlog = logging.getLogger("BitBake.Build")
34parselog = logging.getLogger("BitBake.Parsing")
35providerlog = logging.getLogger("BitBake.Provider")
36
37class NoSpecificMatch(bb.BBHandledException):
38 """
39 Exception raised when no or multiple file matches are found
40 """
41
42class NothingToBuild(Exception):
43 """
44 Exception raised when there is nothing to build
45 """
46
47class CollectionError(bb.BBHandledException):
48 """
49 Exception raised when layer configuration is incorrect
50 """
51
52class state:
Patrick Williamsc0f7c042017-02-23 20:41:17 -060053 initial, parsing, running, shutdown, forceshutdown, stopped, error = list(range(7))
Patrick Williamsc124f4f2015-09-15 14:41:29 -050054
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050055 @classmethod
56 def get_name(cls, code):
57 for name in dir(cls):
58 value = getattr(cls, name)
59 if type(value) == type(cls.initial) and value == code:
60 return name
61 raise ValueError("Invalid status code: %s" % code)
62
Patrick Williamsc124f4f2015-09-15 14:41:29 -050063
64class SkippedPackage:
65 def __init__(self, info = None, reason = None):
66 self.pn = None
67 self.skipreason = None
68 self.provides = None
69 self.rprovides = None
70
71 if info:
72 self.pn = info.pn
73 self.skipreason = info.skipreason
74 self.provides = info.provides
Andrew Geisslerd1e89492021-02-12 15:35:20 -060075 self.rprovides = info.packages + info.rprovides
76 for package in info.packages:
77 self.rprovides += info.rprovides_pkg[package]
Patrick Williamsc124f4f2015-09-15 14:41:29 -050078 elif reason:
79 self.skipreason = reason
80
81
82class CookerFeatures(object):
Andrew Geissler517393d2023-01-13 08:55:19 -060083 _feature_list = [HOB_EXTRA_CACHES, BASEDATASTORE_TRACKING, SEND_SANITYEVENTS, RECIPE_SIGGEN_INFO] = list(range(4))
Patrick Williamsc124f4f2015-09-15 14:41:29 -050084
85 def __init__(self):
86 self._features=set()
87
88 def setFeature(self, f):
89 # validate we got a request for a feature we support
90 if f not in CookerFeatures._feature_list:
91 return
92 self._features.add(f)
93
94 def __contains__(self, f):
95 return f in self._features
96
97 def __iter__(self):
98 return self._features.__iter__()
99
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600100 def __next__(self):
101 return next(self._features)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500102
103
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600104class EventWriter:
105 def __init__(self, cooker, eventfile):
106 self.file_inited = None
107 self.cooker = cooker
108 self.eventfile = eventfile
109 self.event_queue = []
110
111 def write_event(self, event):
112 with open(self.eventfile, "a") as f:
113 try:
114 str_event = codecs.encode(pickle.dumps(event), 'base64').decode('utf-8')
115 f.write("%s\n" % json.dumps({"class": event.__module__ + "." + event.__class__.__name__,
116 "vars": str_event}))
117 except Exception as err:
118 import traceback
119 print(err, traceback.format_exc())
120
121 def send(self, event):
122 if self.file_inited:
123 # we have the file, just write the event
124 self.write_event(event)
125 else:
126 # init on bb.event.BuildStarted
127 name = "%s.%s" % (event.__module__, event.__class__.__name__)
128 if name in ("bb.event.BuildStarted", "bb.cooker.CookerExit"):
129 with open(self.eventfile, "w") as f:
130 f.write("%s\n" % json.dumps({ "allvariables" : self.cooker.getAllKeysWithFlags(["doc", "func"])}))
131
132 self.file_inited = True
133
134 # write pending events
135 for evt in self.event_queue:
136 self.write_event(evt)
137
138 # also write the current event
139 self.write_event(event)
140 else:
141 # queue all events until the file is inited
142 self.event_queue.append(event)
143
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500144#============================================================================#
145# BBCooker
146#============================================================================#
147class BBCooker:
148 """
149 Manages one bitbake build run
150 """
151
Andrew Geissler517393d2023-01-13 08:55:19 -0600152 def __init__(self, featureSet=None, server=None):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600153 self.recipecaches = None
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500154 self.eventlog = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500155 self.skiplist = {}
156 self.featureset = CookerFeatures()
157 if featureSet:
158 for f in featureSet:
159 self.featureset.setFeature(f)
160
Patrick Williams45852732022-04-02 08:58:32 -0500161 self.orig_syspath = sys.path.copy()
162 self.orig_sysmodules = [*sys.modules]
163
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500164 self.configuration = bb.cookerdata.CookerConfiguration()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500165
Andrew Geissler517393d2023-01-13 08:55:19 -0600166 self.process_server = server
167 self.idleCallBackRegister = None
168 self.waitIdle = None
169 if server:
170 self.idleCallBackRegister = server.register_idle_function
171 self.waitIdle = server.wait_for_idle
Andrew Geissler635e0e42020-08-21 15:58:33 -0500172
Brad Bishopf058f492019-01-28 23:50:33 -0500173 bb.debug(1, "BBCooker starting %s" % time.time())
174 sys.stdout.flush()
175
Patrick Williamsde0582f2022-04-08 10:23:27 -0500176 self.configwatcher = None
177 self.confignotifier = None
Brad Bishopf058f492019-01-28 23:50:33 -0500178
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500179 self.watchmask = pyinotify.IN_CLOSE_WRITE | pyinotify.IN_CREATE | pyinotify.IN_DELETE | \
180 pyinotify.IN_DELETE_SELF | pyinotify.IN_MODIFY | pyinotify.IN_MOVE_SELF | \
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500181 pyinotify.IN_MOVED_FROM | pyinotify.IN_MOVED_TO
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500182
Patrick Williamsde0582f2022-04-08 10:23:27 -0500183 self.watcher = None
184 self.notifier = None
Brad Bishopf058f492019-01-28 23:50:33 -0500185
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500186 # If being called by something like tinfoil, we need to clean cached data
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500187 # which may now be invalid
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500188 bb.parse.clear_cache()
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500189 bb.parse.BBHandler.cached_statements = {}
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500190
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500191 self.ui_cmdline = None
Brad Bishop08902b02019-08-20 09:16:51 -0400192 self.hashserv = None
Brad Bishopa34c0302019-09-23 22:34:48 -0400193 self.hashservaddr = None
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500194
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500195 self.inotify_modified_files = []
196
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500197 # TOSTOP must not be set or our children will hang when they output
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600198 try:
199 fd = sys.stdout.fileno()
200 if os.isatty(fd):
201 import termios
202 tcattr = termios.tcgetattr(fd)
203 if tcattr[3] & termios.TOSTOP:
204 buildlog.info("The terminal had the TOSTOP bit set, clearing...")
205 tcattr[3] = tcattr[3] & ~termios.TOSTOP
206 termios.tcsetattr(fd, termios.TCSANOW, tcattr)
207 except UnsupportedOperation:
208 pass
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500209
Andrew Geissler517393d2023-01-13 08:55:19 -0600210 self.command = bb.command.Command(self, self.process_server)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500211 self.state = state.initial
212
213 self.parser = None
214
215 signal.signal(signal.SIGTERM, self.sigterm_exception)
216 # Let SIGHUP exit as SIGTERM
217 signal.signal(signal.SIGHUP, self.sigterm_exception)
218
Brad Bishopf058f492019-01-28 23:50:33 -0500219 bb.debug(1, "BBCooker startup complete %s" % time.time())
220 sys.stdout.flush()
221
Andrew Geissler517393d2023-01-13 08:55:19 -0600222 self.inotify_threadlock = threading.Lock()
223
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500224 def init_configdata(self):
225 if not hasattr(self, "data"):
226 self.initConfigurationData()
227 bb.debug(1, "BBCooker parsed base configuration %s" % time.time())
228 sys.stdout.flush()
229 self.handlePRServ()
230
Patrick Williamsde0582f2022-04-08 10:23:27 -0500231 def setupConfigWatcher(self):
232 if self.configwatcher:
233 self.configwatcher.close()
234 self.confignotifier = None
235 self.configwatcher = None
236 self.configwatcher = pyinotify.WatchManager()
237 self.configwatcher.bbseen = set()
238 self.configwatcher.bbwatchedfiles = set()
239 self.confignotifier = pyinotify.Notifier(self.configwatcher, self.config_notifications)
240
241 def setupParserWatcher(self):
242 if self.watcher:
243 self.watcher.close()
244 self.notifier = None
245 self.watcher = None
246 self.watcher = pyinotify.WatchManager()
247 self.watcher.bbseen = set()
248 self.watcher.bbwatchedfiles = set()
249 self.notifier = pyinotify.Notifier(self.watcher, self.notifications)
250
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500251 def process_inotify_updates(self):
Andrew Geissler517393d2023-01-13 08:55:19 -0600252 with bb.utils.lock_timeout(self.inotify_threadlock):
253 for n in [self.confignotifier, self.notifier]:
254 if n and n.check_events(timeout=0):
255 # read notified events and enqueue them
256 n.read_events()
257
258 def process_inotify_updates_apply(self):
259 with bb.utils.lock_timeout(self.inotify_threadlock):
260 for n in [self.confignotifier, self.notifier]:
261 if n and n.check_events(timeout=0):
262 n.read_events()
263 n.process_events()
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500264
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500265 def config_notifications(self, event):
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500266 if event.maskname == "IN_Q_OVERFLOW":
267 bb.warn("inotify event queue overflowed, invalidating caches.")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500268 self.parsecache_valid = False
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500269 self.baseconfig_valid = False
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500270 bb.parse.clear_cache()
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500271 return
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500272 if not event.pathname in self.configwatcher.bbwatchedfiles:
273 return
Andrew Geissler9aee5002022-03-30 16:27:02 +0000274 if "IN_ISDIR" in event.maskname:
Patrick Williams45852732022-04-02 08:58:32 -0500275 if "IN_CREATE" in event.maskname or "IN_DELETE" in event.maskname:
276 if event.pathname in self.configwatcher.bbseen:
277 self.configwatcher.bbseen.remove(event.pathname)
278 # Could remove all entries starting with the directory but for now...
279 bb.parse.clear_cache()
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500280 if not event.pathname in self.inotify_modified_files:
281 self.inotify_modified_files.append(event.pathname)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500282 self.baseconfig_valid = False
283
284 def notifications(self, event):
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500285 if event.maskname == "IN_Q_OVERFLOW":
286 bb.warn("inotify event queue overflowed, invalidating caches.")
287 self.parsecache_valid = False
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500288 bb.parse.clear_cache()
289 return
290 if event.pathname.endswith("bitbake-cookerdaemon.log") \
291 or event.pathname.endswith("bitbake.lock"):
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500292 return
Andrew Geissler9aee5002022-03-30 16:27:02 +0000293 if "IN_ISDIR" in event.maskname:
Patrick Williams45852732022-04-02 08:58:32 -0500294 if "IN_CREATE" in event.maskname or "IN_DELETE" in event.maskname:
295 if event.pathname in self.watcher.bbseen:
296 self.watcher.bbseen.remove(event.pathname)
297 # Could remove all entries starting with the directory but for now...
298 bb.parse.clear_cache()
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500299 if not event.pathname in self.inotify_modified_files:
300 self.inotify_modified_files.append(event.pathname)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500301 self.parsecache_valid = False
302
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500303 def add_filewatch(self, deps, watcher=None, dirs=False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500304 if not watcher:
305 watcher = self.watcher
306 for i in deps:
Andrew Geissler82c905d2020-04-13 13:39:40 -0500307 watcher.bbwatchedfiles.add(i[0])
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500308 if dirs:
309 f = i[0]
310 else:
311 f = os.path.dirname(i[0])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500312 if f in watcher.bbseen:
313 continue
Andrew Geissler82c905d2020-04-13 13:39:40 -0500314 watcher.bbseen.add(f)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500315 watchtarget = None
316 while True:
317 # We try and add watches for files that don't exist but if they did, would influence
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500318 # the parser. The parent directory of these files may not exist, in which case we need
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500319 # to watch any parent that does exist for changes.
320 try:
321 watcher.add_watch(f, self.watchmask, quiet=False)
322 if watchtarget:
Andrew Geissler82c905d2020-04-13 13:39:40 -0500323 watcher.bbwatchedfiles.add(watchtarget)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500324 break
325 except pyinotify.WatchManagerError as e:
326 if 'ENOENT' in str(e):
327 watchtarget = f
328 f = os.path.dirname(f)
329 if f in watcher.bbseen:
330 break
Andrew Geissler82c905d2020-04-13 13:39:40 -0500331 watcher.bbseen.add(f)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500332 continue
333 if 'ENOSPC' in str(e):
334 providerlog.error("No space left on device or exceeds fs.inotify.max_user_watches?")
335 providerlog.error("To check max_user_watches: sysctl -n fs.inotify.max_user_watches.")
336 providerlog.error("To modify max_user_watches: sysctl -n -w fs.inotify.max_user_watches=<value>.")
337 providerlog.error("Root privilege is required to modify max_user_watches.")
338 raise
339
340 def sigterm_exception(self, signum, stackframe):
341 if signum == signal.SIGTERM:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500342 bb.warn("Cooker received SIGTERM, shutting down...")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500343 elif signum == signal.SIGHUP:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500344 bb.warn("Cooker received SIGHUP, shutting down...")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500345 self.state = state.forceshutdown
346
347 def setFeatures(self, features):
348 # we only accept a new feature set if we're in state initial, so we can reset without problems
349 if not self.state in [state.initial, state.shutdown, state.forceshutdown, state.stopped, state.error]:
350 raise Exception("Illegal state for feature set change")
351 original_featureset = list(self.featureset)
352 for feature in features:
353 self.featureset.setFeature(feature)
354 bb.debug(1, "Features set %s (was %s)" % (original_featureset, list(self.featureset)))
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500355 if (original_featureset != list(self.featureset)) and self.state != state.error and hasattr(self, "data"):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500356 self.reset()
357
358 def initConfigurationData(self):
359
360 self.state = state.initial
361 self.caches_array = []
362
Patrick Williams45852732022-04-02 08:58:32 -0500363 sys.path = self.orig_syspath.copy()
364 for mod in [*sys.modules]:
365 if mod not in self.orig_sysmodules:
366 del sys.modules[mod]
367
Patrick Williamsde0582f2022-04-08 10:23:27 -0500368 self.setupConfigWatcher()
369
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500370 # Need to preserve BB_CONSOLELOG over resets
371 consolelog = None
372 if hasattr(self, "data"):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500373 consolelog = self.data.getVar("BB_CONSOLELOG")
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500374
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500375 if CookerFeatures.BASEDATASTORE_TRACKING in self.featureset:
376 self.enableDataTracking()
377
Andrew Geissler517393d2023-01-13 08:55:19 -0600378 caches_name_array = ['bb.cache:CoreRecipeInfo']
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500379 # We hardcode all known cache types in a single place, here.
380 if CookerFeatures.HOB_EXTRA_CACHES in self.featureset:
Andrew Geissler517393d2023-01-13 08:55:19 -0600381 caches_name_array.append("bb.cache_extra:HobRecipeInfo")
382 if CookerFeatures.RECIPE_SIGGEN_INFO in self.featureset:
383 caches_name_array.append("bb.cache:SiggenRecipeInfo")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500384
385 # At least CoreRecipeInfo will be loaded, so caches_array will never be empty!
386 # This is the entry point, no further check needed!
387 for var in caches_name_array:
388 try:
389 module_name, cache_name = var.split(':')
390 module = __import__(module_name, fromlist=(cache_name,))
391 self.caches_array.append(getattr(module, cache_name))
392 except ImportError as exc:
393 logger.critical("Unable to import extra RecipeInfo '%s' from '%s': %s" % (cache_name, module_name, exc))
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500394 raise bb.BBHandledException()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500395
396 self.databuilder = bb.cookerdata.CookerDataBuilder(self.configuration, False)
397 self.databuilder.parseBaseConfiguration()
398 self.data = self.databuilder.data
399 self.data_hash = self.databuilder.data_hash
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500400 self.extraconfigdata = {}
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500401
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500402 if consolelog:
403 self.data.setVar("BB_CONSOLELOG", consolelog)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500404
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500405 self.data.setVar('BB_CMDLINE', self.ui_cmdline)
406
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500407 if CookerFeatures.BASEDATASTORE_TRACKING in self.featureset:
408 self.disableDataTracking()
409
Brad Bishop15ae2502019-06-18 21:44:24 -0400410 for mc in self.databuilder.mcdata.values():
Brad Bishop15ae2502019-06-18 21:44:24 -0400411 self.add_filewatch(mc.getVar("__base_depends", False), self.configwatcher)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500412
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500413 self.baseconfig_valid = True
414 self.parsecache_valid = False
415
416 def handlePRServ(self):
417 # Setup a PR Server based on the new configuration
418 try:
419 self.prhost = prserv.serv.auto_start(self.data)
420 except prserv.serv.PRServiceConfigError as e:
Andrew Geisslerd159c7f2021-09-02 21:05:58 -0500421 bb.fatal("Unable to start PR Server, exiting, check the bitbake-cookerdaemon.log")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500422
Brad Bishopa34c0302019-09-23 22:34:48 -0400423 if self.data.getVar("BB_HASHSERVE") == "auto":
424 # Create a new hash server bound to a unix domain socket
Brad Bishop08902b02019-08-20 09:16:51 -0400425 if not self.hashserv:
426 dbfile = (self.data.getVar("PERSISTENT_DIR") or self.data.getVar("CACHE")) + "/hashserv.db"
Andrew Geissler595f6302022-01-24 19:11:47 +0000427 upstream = self.data.getVar("BB_HASHSERVE_UPSTREAM") or None
428 if upstream:
429 import socket
430 try:
431 sock = socket.create_connection(upstream.split(":"), 5)
432 sock.close()
433 except socket.error as e:
Andrew Geissler87f5cff2022-09-30 13:13:31 -0500434 bb.warn("BB_HASHSERVE_UPSTREAM is not valid, unable to connect hash equivalence server at '%s': %s"
Andrew Geissler595f6302022-01-24 19:11:47 +0000435 % (upstream, repr(e)))
436
Brad Bishopa34c0302019-09-23 22:34:48 -0400437 self.hashservaddr = "unix://%s/hashserve.sock" % self.data.getVar("TOPDIR")
Andrew Geissler5199d832021-09-24 16:47:35 -0500438 self.hashserv = hashserv.create_server(
439 self.hashservaddr,
440 dbfile,
441 sync=False,
Andrew Geissler595f6302022-01-24 19:11:47 +0000442 upstream=upstream,
Andrew Geissler5199d832021-09-24 16:47:35 -0500443 )
Patrick Williams213cb262021-08-07 19:21:33 -0500444 self.hashserv.serve_as_process()
Brad Bishop08902b02019-08-20 09:16:51 -0400445 for mc in self.databuilder.mcdata:
Andrew Geissler517393d2023-01-13 08:55:19 -0600446 self.databuilder.mcorigdata[mc].setVar("BB_HASHSERVE", self.hashservaddr)
Brad Bishopa34c0302019-09-23 22:34:48 -0400447 self.databuilder.mcdata[mc].setVar("BB_HASHSERVE", self.hashservaddr)
Brad Bishop08902b02019-08-20 09:16:51 -0400448
449 bb.parse.init_parser(self.data)
450
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500451 def enableDataTracking(self):
452 self.configuration.tracking = True
453 if hasattr(self, "data"):
454 self.data.enableTracking()
455
456 def disableDataTracking(self):
457 self.configuration.tracking = False
458 if hasattr(self, "data"):
459 self.data.disableTracking()
460
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500461 def parseConfiguration(self):
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600462 self.updateCacheSync()
463
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500464 # Change nice level if we're asked to
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500465 nice = self.data.getVar("BB_NICE_LEVEL")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500466 if nice:
467 curnice = os.nice(0)
468 nice = int(nice) - curnice
469 buildlog.verbose("Renice to %s " % os.nice(nice))
470
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600471 if self.recipecaches:
472 del self.recipecaches
473 self.multiconfigs = self.databuilder.mcdata.keys()
474 self.recipecaches = {}
475 for mc in self.multiconfigs:
476 self.recipecaches[mc] = bb.cache.CacheData(self.caches_array)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500477
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500478 self.handleCollections(self.data.getVar("BBFILE_COLLECTIONS"))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500479
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500480 self.parsecache_valid = False
481
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500482 def updateConfigOpts(self, options, environment, cmdline):
483 self.ui_cmdline = cmdline
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500484 clean = True
485 for o in options:
486 if o in ['prefile', 'postfile']:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500487 # Only these options may require a reparse
488 try:
489 if getattr(self.configuration, o) == options[o]:
490 # Value is the same, no need to mark dirty
491 continue
492 except AttributeError:
493 pass
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600494 logger.debug("Marking as dirty due to '%s' option change to '%s'" % (o, options[o]))
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500495 print("Marking as dirty due to '%s' option change to '%s'" % (o, options[o]))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500496 clean = False
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500497 if hasattr(self.configuration, o):
498 setattr(self.configuration, o, options[o])
499
500 if self.configuration.writeeventlog:
501 if self.eventlog and self.eventlog[0] != self.configuration.writeeventlog:
502 bb.event.unregister_UIHhandler(self.eventlog[1])
503 if not self.eventlog or self.eventlog[0] != self.configuration.writeeventlog:
504 # we log all events to a file if so directed
505 # register the log file writer as UI Handler
506 writer = EventWriter(self, self.configuration.writeeventlog)
507 EventLogWriteHandler = namedtuple('EventLogWriteHandler', ['event'])
508 self.eventlog = (self.configuration.writeeventlog, bb.event.register_UIHhandler(EventLogWriteHandler(writer)))
509
510 bb.msg.loggerDefaultLogLevel = self.configuration.default_loglevel
511 bb.msg.loggerDefaultDomains = self.configuration.debug_domains
512
513 if hasattr(self, "data"):
514 origenv = bb.data.init()
515 for k in environment:
516 origenv.setVar(k, environment[k])
517 self.data.setVar("BB_ORIGENV", origenv)
518
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500519 for k in bb.utils.approved_variables():
520 if k in environment and k not in self.configuration.env:
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600521 logger.debug("Updating new environment variable %s to %s" % (k, environment[k]))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500522 self.configuration.env[k] = environment[k]
523 clean = False
524 if k in self.configuration.env and k not in environment:
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600525 logger.debug("Updating environment variable %s (deleted)" % (k))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500526 del self.configuration.env[k]
527 clean = False
528 if k not in self.configuration.env and k not in environment:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500529 continue
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500530 if environment[k] != self.configuration.env[k]:
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600531 logger.debug("Updating environment variable %s from %s to %s" % (k, self.configuration.env[k], environment[k]))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500532 self.configuration.env[k] = environment[k]
533 clean = False
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500534
535 # Now update all the variables not in the datastore to match
536 self.configuration.env = environment
537
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500538 if not clean:
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600539 logger.debug("Base environment change, triggering reparse")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500540 self.reset()
541
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500542 def showVersions(self):
543
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500544 (latest_versions, preferred_versions, required) = self.findProviders()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500545
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500546 logger.plain("%-35s %25s %25s %25s", "Recipe Name", "Latest Version", "Preferred Version", "Required Version")
547 logger.plain("%-35s %25s %25s %25s\n", "===========", "==============", "=================", "================")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500548
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500549 for p in sorted(self.recipecaches[''].pkg_pn):
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500550 preferred = preferred_versions[p]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500551 latest = latest_versions[p]
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500552 requiredstr = ""
553 preferredstr = ""
554 if required[p]:
555 if preferred[0] is not None:
556 requiredstr = preferred[0][0] + ":" + preferred[0][1] + '-' + preferred[0][2]
557 else:
558 bb.fatal("REQUIRED_VERSION of package %s not available" % p)
559 else:
560 preferredstr = preferred[0][0] + ":" + preferred[0][1] + '-' + preferred[0][2]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500561
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500562 lateststr = latest[0][0] + ":" + latest[0][1] + "-" + latest[0][2]
563
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500564 if preferred == latest:
565 preferredstr = ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500566
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500567 logger.plain("%-35s %25s %25s %25s", p, lateststr, preferredstr, requiredstr)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500568
569 def showEnvironment(self, buildfile=None, pkgs_to_build=None):
570 """
571 Show the outer or per-recipe environment
572 """
573 fn = None
574 envdata = None
Brad Bishop15ae2502019-06-18 21:44:24 -0400575 mc = ''
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500576 if not pkgs_to_build:
577 pkgs_to_build = []
578
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500579 orig_tracking = self.configuration.tracking
580 if not orig_tracking:
581 self.enableDataTracking()
582 self.reset()
Andrew Geissler9aee5002022-03-30 16:27:02 +0000583 # reset() resets to the UI requested value so we have to redo this
584 self.enableDataTracking()
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500585
Brad Bishop15ae2502019-06-18 21:44:24 -0400586 def mc_base(p):
587 if p.startswith('mc:'):
588 s = p.split(':')
589 if len(s) == 2:
590 return s[1]
591 return None
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500592
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500593 if buildfile:
594 # Parse the configuration here. We need to do it explicitly here since
595 # this showEnvironment() code path doesn't use the cache
596 self.parseConfiguration()
597
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600598 fn, cls, mc = bb.cache.virtualfn2realfn(buildfile)
Andrew Geissler5a43b432020-06-13 10:46:56 -0500599 fn = self.matchFile(fn, mc)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600600 fn = bb.cache.realfn2virtual(fn, cls, mc)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500601 elif len(pkgs_to_build) == 1:
Brad Bishop15ae2502019-06-18 21:44:24 -0400602 mc = mc_base(pkgs_to_build[0])
603 if not mc:
604 ignore = self.data.getVar("ASSUME_PROVIDED") or ""
605 if pkgs_to_build[0] in set(ignore.split()):
606 bb.fatal("%s is in ASSUME_PROVIDED" % pkgs_to_build[0])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500607
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000608 taskdata, runlist = self.buildTaskData(pkgs_to_build, None, self.configuration.halt, allowincomplete=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500609
Brad Bishop15ae2502019-06-18 21:44:24 -0400610 mc = runlist[0][0]
611 fn = runlist[0][3]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500612
613 if fn:
614 try:
Andrew Geissler517393d2023-01-13 08:55:19 -0600615 envdata = self.databuilder.parseRecipe(fn, self.collections[mc].get_file_appends(fn))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500616 except Exception as e:
617 parselog.exception("Unable to read %s", fn)
618 raise
Brad Bishop15ae2502019-06-18 21:44:24 -0400619 else:
620 if not mc in self.databuilder.mcdata:
621 bb.fatal('Not multiconfig named "%s" found' % mc)
622 envdata = self.databuilder.mcdata[mc]
623 data.expandKeys(envdata)
624 parse.ast.runAnonFuncs(envdata)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500625
626 # Display history
627 with closing(StringIO()) as env:
628 self.data.inchistory.emit(env)
629 logger.plain(env.getvalue())
630
631 # emit variables and shell functions
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500632 with closing(StringIO()) as env:
633 data.emit_env(env, envdata, True)
634 logger.plain(env.getvalue())
635
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000636 # emit the metadata which isn't valid shell
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500637 for e in sorted(envdata.keys()):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600638 if envdata.getVarFlag(e, 'func', False) and envdata.getVarFlag(e, 'python', False):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500639 logger.plain("\npython %s () {\n%s}\n", e, envdata.getVar(e, False))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500640
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500641 if not orig_tracking:
642 self.disableDataTracking()
643 self.reset()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500644
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000645 def buildTaskData(self, pkgs_to_build, task, halt, allowincomplete=False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500646 """
647 Prepare a runqueue and taskdata object for iteration over pkgs_to_build
648 """
649 bb.event.fire(bb.event.TreeDataPreparationStarted(), self.data)
650
651 # A task of None means use the default task
652 if task is None:
653 task = self.configuration.cmd
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500654 if not task.startswith("do_"):
655 task = "do_%s" % task
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500656
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500657 targetlist = self.checkPackages(pkgs_to_build, task)
658 fulltargetlist = []
659 defaulttask_implicit = ''
660 defaulttask_explicit = False
661 wildcard = False
662
663 # Wild card expansion:
Brad Bishop15ae2502019-06-18 21:44:24 -0400664 # Replace string such as "mc:*:bash"
665 # into "mc:A:bash mc:B:bash bash"
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500666 for k in targetlist:
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600667 if k.startswith("mc:") and k.count(':') >= 2:
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500668 if wildcard:
669 bb.fatal('multiconfig conflict')
670 if k.split(":")[1] == "*":
671 wildcard = True
672 for mc in self.multiconfigs:
673 if mc:
674 fulltargetlist.append(k.replace('*', mc))
675 # implicit default task
676 else:
677 defaulttask_implicit = k.split(":")[2]
678 else:
679 fulltargetlist.append(k)
680 else:
681 defaulttask_explicit = True
682 fulltargetlist.append(k)
683
684 if not defaulttask_explicit and defaulttask_implicit != '':
685 fulltargetlist.append(defaulttask_implicit)
686
687 bb.debug(1,"Target list: %s" % (str(fulltargetlist)))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600688 taskdata = {}
689 localdata = {}
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500690
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600691 for mc in self.multiconfigs:
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000692 taskdata[mc] = bb.taskdata.TaskData(halt, skiplist=self.skiplist, allowincomplete=allowincomplete)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600693 localdata[mc] = data.createCopy(self.databuilder.mcdata[mc])
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600694 bb.data.expandKeys(localdata[mc])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500695
696 current = 0
697 runlist = []
698 for k in fulltargetlist:
Andrew Geisslerb7d28612020-07-24 16:15:54 -0500699 origk = k
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600700 mc = ""
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600701 if k.startswith("mc:") and k.count(':') >= 2:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600702 mc = k.split(":")[1]
703 k = ":".join(k.split(":")[2:])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500704 ktask = task
705 if ":do_" in k:
706 k2 = k.split(":do_")
707 k = k2[0]
708 ktask = k2[1]
Andrew Geisslerb7d28612020-07-24 16:15:54 -0500709
710 if mc not in self.multiconfigs:
711 bb.fatal("Multiconfig dependency %s depends on nonexistent multiconfig configuration named %s" % (origk, mc))
712
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600713 taskdata[mc].add_provider(localdata[mc], self.recipecaches[mc], k)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500714 current += 1
715 if not ktask.startswith("do_"):
716 ktask = "do_%s" % ktask
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600717 if k not in taskdata[mc].build_targets or not taskdata[mc].build_targets[k]:
718 # e.g. in ASSUME_PROVIDED
719 continue
720 fn = taskdata[mc].build_targets[k][0]
721 runlist.append([mc, k, ktask, fn])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500722 bb.event.fire(bb.event.TreeDataPreparationProgress(current, len(fulltargetlist)), self.data)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600723
Brad Bishop15ae2502019-06-18 21:44:24 -0400724 havemc = False
725 for mc in self.multiconfigs:
726 if taskdata[mc].get_mcdepends():
727 havemc = True
Brad Bishopf058f492019-01-28 23:50:33 -0500728
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800729 # No need to do check providers if there are no mcdeps or not an mc build
Brad Bishop15ae2502019-06-18 21:44:24 -0400730 if havemc or len(self.multiconfigs) > 1:
Andrew Geissler99467da2019-02-25 18:54:23 -0600731 seen = set()
732 new = True
733 # Make sure we can provide the multiconfig dependency
734 while new:
735 mcdeps = set()
736 # Add unresolved first, so we can get multiconfig indirect dependencies on time
737 for mc in self.multiconfigs:
738 taskdata[mc].add_unresolved(localdata[mc], self.recipecaches[mc])
739 mcdeps |= set(taskdata[mc].get_mcdepends())
740 new = False
Patrick Williams03907ee2022-05-01 06:28:52 -0500741 for k in mcdeps:
742 if k in seen:
743 continue
744 l = k.split(':')
745 depmc = l[2]
746 if depmc not in self.multiconfigs:
747 bb.fatal("Multiconfig dependency %s depends on nonexistent multiconfig configuration named configuration %s" % (k,depmc))
748 else:
749 logger.debug("Adding providers for multiconfig dependency %s" % l[3])
750 taskdata[depmc].add_provider(localdata[depmc], self.recipecaches[depmc], l[3])
751 seen.add(k)
752 new = True
Brad Bishopf058f492019-01-28 23:50:33 -0500753
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600754 for mc in self.multiconfigs:
755 taskdata[mc].add_unresolved(localdata[mc], self.recipecaches[mc])
756
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500757 bb.event.fire(bb.event.TreeDataPreparationCompleted(len(fulltargetlist)), self.data)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600758 return taskdata, runlist
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500759
760 def prepareTreeData(self, pkgs_to_build, task):
761 """
762 Prepare a runqueue and taskdata object for iteration over pkgs_to_build
763 """
764
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000765 # We set halt to False here to prevent unbuildable targets raising
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500766 # an exception when we're just generating data
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600767 taskdata, runlist = self.buildTaskData(pkgs_to_build, task, False, allowincomplete=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500768
769 return runlist, taskdata
770
771 ######## WARNING : this function requires cache_extra to be enabled ########
772
773 def generateTaskDepTreeData(self, pkgs_to_build, task):
774 """
775 Create a dependency graph of pkgs_to_build including reverse dependency
776 information.
777 """
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500778 if not task.startswith("do_"):
779 task = "do_%s" % task
780
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500781 runlist, taskdata = self.prepareTreeData(pkgs_to_build, task)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600782 rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500783 rq.rqdata.prepare()
784 return self.buildDependTree(rq, taskdata)
785
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600786 @staticmethod
787 def add_mc_prefix(mc, pn):
788 if mc:
Brad Bishop15ae2502019-06-18 21:44:24 -0400789 return "mc:%s:%s" % (mc, pn)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600790 return pn
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500791
792 def buildDependTree(self, rq, taskdata):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600793 seen_fns = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500794 depend_tree = {}
795 depend_tree["depends"] = {}
796 depend_tree["tdepends"] = {}
797 depend_tree["pn"] = {}
798 depend_tree["rdepends-pn"] = {}
799 depend_tree["packages"] = {}
800 depend_tree["rdepends-pkg"] = {}
801 depend_tree["rrecs-pkg"] = {}
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500802 depend_tree['providermap'] = {}
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600803 depend_tree["layer-priorities"] = self.bbfile_config_priorities
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500804
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600805 for mc in taskdata:
806 for name, fn in list(taskdata[mc].get_providermap().items()):
807 pn = self.recipecaches[mc].pkg_fn[fn]
808 pn = self.add_mc_prefix(mc, pn)
809 if name != pn:
810 version = "%s:%s-%s" % self.recipecaches[mc].pkg_pepvpr[fn]
811 depend_tree['providermap'][name] = (pn, version)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500812
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600813 for tid in rq.rqdata.runtaskentries:
814 (mc, fn, taskname, taskfn) = bb.runqueue.split_tid_mcfn(tid)
815 pn = self.recipecaches[mc].pkg_fn[taskfn]
816 pn = self.add_mc_prefix(mc, pn)
817 version = "%s:%s-%s" % self.recipecaches[mc].pkg_pepvpr[taskfn]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500818 if pn not in depend_tree["pn"]:
819 depend_tree["pn"][pn] = {}
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600820 depend_tree["pn"][pn]["filename"] = taskfn
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500821 depend_tree["pn"][pn]["version"] = version
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600822 depend_tree["pn"][pn]["inherits"] = self.recipecaches[mc].inherits.get(taskfn, None)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500823
824 # if we have extra caches, list all attributes they bring in
825 extra_info = []
826 for cache_class in self.caches_array:
827 if type(cache_class) is type and issubclass(cache_class, bb.cache.RecipeInfoCommon) and hasattr(cache_class, 'cachefields'):
828 cachefields = getattr(cache_class, 'cachefields', [])
829 extra_info = extra_info + cachefields
830
831 # for all attributes stored, add them to the dependency tree
832 for ei in extra_info:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600833 depend_tree["pn"][pn][ei] = vars(self.recipecaches[mc])[ei][taskfn]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500834
835
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500836 dotname = "%s.%s" % (pn, bb.runqueue.taskname_from_tid(tid))
837 if not dotname in depend_tree["tdepends"]:
838 depend_tree["tdepends"][dotname] = []
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600839 for dep in rq.rqdata.runtaskentries[tid].depends:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800840 (depmc, depfn, _, deptaskfn) = bb.runqueue.split_tid_mcfn(dep)
841 deppn = self.recipecaches[depmc].pkg_fn[deptaskfn]
Andrew Geissler595f6302022-01-24 19:11:47 +0000842 if depmc:
843 depmc = "mc:" + depmc + ":"
844 depend_tree["tdepends"][dotname].append("%s%s.%s" % (depmc, deppn, bb.runqueue.taskname_from_tid(dep)))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600845 if taskfn not in seen_fns:
846 seen_fns.append(taskfn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500847 packages = []
848
849 depend_tree["depends"][pn] = []
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600850 for dep in taskdata[mc].depids[taskfn]:
851 depend_tree["depends"][pn].append(dep)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500852
853 depend_tree["rdepends-pn"][pn] = []
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600854 for rdep in taskdata[mc].rdepids[taskfn]:
855 depend_tree["rdepends-pn"][pn].append(rdep)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500856
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600857 rdepends = self.recipecaches[mc].rundeps[taskfn]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500858 for package in rdepends:
859 depend_tree["rdepends-pkg"][package] = []
860 for rdepend in rdepends[package]:
861 depend_tree["rdepends-pkg"][package].append(rdepend)
862 packages.append(package)
863
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600864 rrecs = self.recipecaches[mc].runrecs[taskfn]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500865 for package in rrecs:
866 depend_tree["rrecs-pkg"][package] = []
867 for rdepend in rrecs[package]:
868 depend_tree["rrecs-pkg"][package].append(rdepend)
869 if not package in packages:
870 packages.append(package)
871
872 for package in packages:
873 if package not in depend_tree["packages"]:
874 depend_tree["packages"][package] = {}
875 depend_tree["packages"][package]["pn"] = pn
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600876 depend_tree["packages"][package]["filename"] = taskfn
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500877 depend_tree["packages"][package]["version"] = version
878
879 return depend_tree
880
881 ######## WARNING : this function requires cache_extra to be enabled ########
882 def generatePkgDepTreeData(self, pkgs_to_build, task):
883 """
884 Create a dependency tree of pkgs_to_build, returning the data.
885 """
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500886 if not task.startswith("do_"):
887 task = "do_%s" % task
888
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500889 _, taskdata = self.prepareTreeData(pkgs_to_build, task)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500890
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600891 seen_fns = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500892 depend_tree = {}
893 depend_tree["depends"] = {}
894 depend_tree["pn"] = {}
895 depend_tree["rdepends-pn"] = {}
896 depend_tree["rdepends-pkg"] = {}
897 depend_tree["rrecs-pkg"] = {}
898
899 # if we have extra caches, list all attributes they bring in
900 extra_info = []
901 for cache_class in self.caches_array:
902 if type(cache_class) is type and issubclass(cache_class, bb.cache.RecipeInfoCommon) and hasattr(cache_class, 'cachefields'):
903 cachefields = getattr(cache_class, 'cachefields', [])
904 extra_info = extra_info + cachefields
905
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600906 tids = []
907 for mc in taskdata:
908 for tid in taskdata[mc].taskentries:
909 tids.append(tid)
910
911 for tid in tids:
912 (mc, fn, taskname, taskfn) = bb.runqueue.split_tid_mcfn(tid)
913
914 pn = self.recipecaches[mc].pkg_fn[taskfn]
915 pn = self.add_mc_prefix(mc, pn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500916
917 if pn not in depend_tree["pn"]:
918 depend_tree["pn"][pn] = {}
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600919 depend_tree["pn"][pn]["filename"] = taskfn
920 version = "%s:%s-%s" % self.recipecaches[mc].pkg_pepvpr[taskfn]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500921 depend_tree["pn"][pn]["version"] = version
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600922 rdepends = self.recipecaches[mc].rundeps[taskfn]
923 rrecs = self.recipecaches[mc].runrecs[taskfn]
924 depend_tree["pn"][pn]["inherits"] = self.recipecaches[mc].inherits.get(taskfn, None)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500925
926 # for all extra attributes stored, add them to the dependency tree
927 for ei in extra_info:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600928 depend_tree["pn"][pn][ei] = vars(self.recipecaches[mc])[ei][taskfn]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500929
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600930 if taskfn not in seen_fns:
931 seen_fns.append(taskfn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500932
933 depend_tree["depends"][pn] = []
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500934 for dep in taskdata[mc].depids[taskfn]:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500935 pn_provider = ""
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600936 if dep in taskdata[mc].build_targets and taskdata[mc].build_targets[dep]:
937 fn_provider = taskdata[mc].build_targets[dep][0]
938 pn_provider = self.recipecaches[mc].pkg_fn[fn_provider]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500939 else:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500940 pn_provider = dep
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600941 pn_provider = self.add_mc_prefix(mc, pn_provider)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500942 depend_tree["depends"][pn].append(pn_provider)
943
944 depend_tree["rdepends-pn"][pn] = []
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600945 for rdep in taskdata[mc].rdepids[taskfn]:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500946 pn_rprovider = ""
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600947 if rdep in taskdata[mc].run_targets and taskdata[mc].run_targets[rdep]:
948 fn_rprovider = taskdata[mc].run_targets[rdep][0]
949 pn_rprovider = self.recipecaches[mc].pkg_fn[fn_rprovider]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500950 else:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600951 pn_rprovider = rdep
952 pn_rprovider = self.add_mc_prefix(mc, pn_rprovider)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500953 depend_tree["rdepends-pn"][pn].append(pn_rprovider)
954
955 depend_tree["rdepends-pkg"].update(rdepends)
956 depend_tree["rrecs-pkg"].update(rrecs)
957
958 return depend_tree
959
960 def generateDepTreeEvent(self, pkgs_to_build, task):
961 """
962 Create a task dependency graph of pkgs_to_build.
963 Generate an event with the result
964 """
965 depgraph = self.generateTaskDepTreeData(pkgs_to_build, task)
966 bb.event.fire(bb.event.DepTreeGenerated(depgraph), self.data)
967
968 def generateDotGraphFiles(self, pkgs_to_build, task):
969 """
970 Create a task dependency graph of pkgs_to_build.
971 Save the result to a set of .dot files.
972 """
973
974 depgraph = self.generateTaskDepTreeData(pkgs_to_build, task)
975
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500976 with open('pn-buildlist', 'w') as f:
977 for pn in depgraph["pn"]:
978 f.write(pn + "\n")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500979 logger.info("PN build list saved to 'pn-buildlist'")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500980
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500981 # Remove old format output files to ensure no confusion with stale data
982 try:
983 os.unlink('pn-depends.dot')
984 except FileNotFoundError:
985 pass
986 try:
987 os.unlink('package-depends.dot')
988 except FileNotFoundError:
989 pass
Brad Bishop79641f22019-09-10 07:20:22 -0400990 try:
991 os.unlink('recipe-depends.dot')
992 except FileNotFoundError:
993 pass
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500994
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500995 with open('task-depends.dot', 'w') as f:
996 f.write("digraph depends {\n")
Brad Bishop316dfdd2018-06-25 12:45:53 -0400997 for task in sorted(depgraph["tdepends"]):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500998 (pn, taskname) = task.rsplit(".", 1)
999 fn = depgraph["pn"][pn]["filename"]
1000 version = depgraph["pn"][pn]["version"]
1001 f.write('"%s.%s" [label="%s %s\\n%s\\n%s"]\n' % (pn, taskname, pn, taskname, version, fn))
Brad Bishop316dfdd2018-06-25 12:45:53 -04001002 for dep in sorted(depgraph["tdepends"][task]):
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001003 f.write('"%s" -> "%s"\n' % (task, dep))
1004 f.write("}\n")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001005 logger.info("Task dependencies saved to 'task-depends.dot'")
1006
1007 def show_appends_with_no_recipes(self):
Andrew Geissler5a43b432020-06-13 10:46:56 -05001008 appends_without_recipes = {}
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001009 # Determine which bbappends haven't been applied
Andrew Geissler5a43b432020-06-13 10:46:56 -05001010 for mc in self.multiconfigs:
1011 # First get list of recipes, including skipped
1012 recipefns = list(self.recipecaches[mc].pkg_fn.keys())
1013 recipefns.extend(self.skiplist.keys())
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001014
Andrew Geissler5a43b432020-06-13 10:46:56 -05001015 # Work out list of bbappends that have been applied
1016 applied_appends = []
1017 for fn in recipefns:
1018 applied_appends.extend(self.collections[mc].get_file_appends(fn))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001019
Andrew Geissler5a43b432020-06-13 10:46:56 -05001020 appends_without_recipes[mc] = []
1021 for _, appendfn in self.collections[mc].bbappends:
1022 if not appendfn in applied_appends:
1023 appends_without_recipes[mc].append(appendfn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001024
Andrew Geissler5a43b432020-06-13 10:46:56 -05001025 msgs = []
1026 for mc in sorted(appends_without_recipes.keys()):
1027 if appends_without_recipes[mc]:
1028 msgs.append('No recipes in %s available for:\n %s' % (mc if mc else 'default',
1029 '\n '.join(appends_without_recipes[mc])))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001030
Andrew Geissler5a43b432020-06-13 10:46:56 -05001031 if msgs:
1032 msg = "\n".join(msgs)
1033 warn_only = self.databuilder.mcdata[mc].getVar("BB_DANGLINGAPPENDS_WARNONLY", \
1034 False) or "no"
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001035 if warn_only.lower() in ("1", "yes", "true"):
1036 bb.warn(msg)
1037 else:
1038 bb.fatal(msg)
1039
1040 def handlePrefProviders(self):
1041
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001042 for mc in self.multiconfigs:
1043 localdata = data.createCopy(self.databuilder.mcdata[mc])
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001044 bb.data.expandKeys(localdata)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001045
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001046 # Handle PREFERRED_PROVIDERS
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001047 for p in (localdata.getVar('PREFERRED_PROVIDERS') or "").split():
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001048 try:
1049 (providee, provider) = p.split(':')
1050 except:
1051 providerlog.critical("Malformed option in PREFERRED_PROVIDERS variable: %s" % p)
1052 continue
1053 if providee in self.recipecaches[mc].preferred and self.recipecaches[mc].preferred[providee] != provider:
1054 providerlog.error("conflicting preferences for %s: both %s and %s specified", providee, provider, self.recipecaches[mc].preferred[providee])
1055 self.recipecaches[mc].preferred[providee] = provider
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001056
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001057 def findConfigFilePath(self, configfile):
1058 """
1059 Find the location on disk of configfile and if it exists and was parsed by BitBake
1060 emit the ConfigFilePathFound event with the path to the file.
1061 """
1062 path = bb.cookerdata.findConfigFile(configfile, self.data)
1063 if not path:
1064 return
1065
1066 # Generate a list of parsed configuration files by searching the files
1067 # listed in the __depends and __base_depends variables with a .conf suffix.
1068 conffiles = []
1069 dep_files = self.data.getVar('__base_depends', False) or []
1070 dep_files = dep_files + (self.data.getVar('__depends', False) or [])
1071
1072 for f in dep_files:
1073 if f[0].endswith(".conf"):
1074 conffiles.append(f[0])
1075
1076 _, conf, conffile = path.rpartition("conf/")
1077 match = os.path.join(conf, conffile)
1078 # Try and find matches for conf/conffilename.conf as we don't always
1079 # have the full path to the file.
1080 for cfg in conffiles:
1081 if cfg.endswith(match):
1082 bb.event.fire(bb.event.ConfigFilePathFound(path),
1083 self.data)
1084 break
1085
1086 def findFilesMatchingInDir(self, filepattern, directory):
1087 """
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001088 Searches for files containing the substring 'filepattern' which are children of
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001089 'directory' in each BBPATH. i.e. to find all rootfs package classes available
1090 to BitBake one could call findFilesMatchingInDir(self, 'rootfs_', 'classes')
1091 or to find all machine configuration files one could call:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001092 findFilesMatchingInDir(self, '.conf', 'conf/machine')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001093 """
1094
1095 matches = []
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001096 bbpaths = self.data.getVar('BBPATH').split(':')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001097 for path in bbpaths:
1098 dirpath = os.path.join(path, directory)
1099 if os.path.exists(dirpath):
1100 for root, dirs, files in os.walk(dirpath):
1101 for f in files:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001102 if filepattern in f:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001103 matches.append(f)
1104
1105 if matches:
1106 bb.event.fire(bb.event.FilesMatchingFound(filepattern, matches), self.data)
1107
Patrick Williams93c203f2021-10-06 16:15:23 -05001108 def testCookerCommandEvent(self, filepattern):
1109 # Dummy command used by OEQA selftest to test tinfoil without IO
1110 matches = ["A", "B"]
1111 bb.event.fire(bb.event.FilesMatchingFound(filepattern, matches), self.data)
1112
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001113 def findProviders(self, mc=''):
Andrew Geissler82c905d2020-04-13 13:39:40 -05001114 return bb.providers.findProviders(self.databuilder.mcdata[mc], self.recipecaches[mc], self.recipecaches[mc].pkg_pn)
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001115
1116 def findBestProvider(self, pn, mc=''):
1117 if pn in self.recipecaches[mc].providers:
1118 filenames = self.recipecaches[mc].providers[pn]
Andrew Geissler82c905d2020-04-13 13:39:40 -05001119 eligible, foundUnique = bb.providers.filterProviders(filenames, pn, self.databuilder.mcdata[mc], self.recipecaches[mc])
Andrew Geissler95ac1b82021-03-31 14:34:31 -05001120 if eligible is not None:
1121 filename = eligible[0]
1122 else:
1123 filename = None
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001124 return None, None, None, filename
1125 elif pn in self.recipecaches[mc].pkg_pn:
Andrew Geissler95ac1b82021-03-31 14:34:31 -05001126 (latest, latest_f, preferred_ver, preferred_file, required) = bb.providers.findBestProvider(pn, self.databuilder.mcdata[mc], self.recipecaches[mc], self.recipecaches[mc].pkg_pn)
1127 if required and preferred_file is None:
1128 return None, None, None, None
1129 return (latest, latest_f, preferred_ver, preferred_file)
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001130 else:
1131 return None, None, None, None
1132
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001133 def findConfigFiles(self, varname):
1134 """
1135 Find config files which are appropriate values for varname.
1136 i.e. MACHINE, DISTRO
1137 """
1138 possible = []
1139 var = varname.lower()
1140
1141 data = self.data
1142 # iterate configs
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001143 bbpaths = data.getVar('BBPATH').split(':')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001144 for path in bbpaths:
1145 confpath = os.path.join(path, "conf", var)
1146 if os.path.exists(confpath):
1147 for root, dirs, files in os.walk(confpath):
1148 # get all child files, these are appropriate values
1149 for f in files:
1150 val, sep, end = f.rpartition('.')
1151 if end == 'conf':
1152 possible.append(val)
1153
1154 if possible:
1155 bb.event.fire(bb.event.ConfigFilesFound(var, possible), self.data)
1156
1157 def findInheritsClass(self, klass):
1158 """
1159 Find all recipes which inherit the specified class
1160 """
1161 pkg_list = []
1162
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001163 for pfn in self.recipecaches[''].pkg_fn:
1164 inherits = self.recipecaches[''].inherits.get(pfn, None)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001165 if inherits and klass in inherits:
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001166 pkg_list.append(self.recipecaches[''].pkg_fn[pfn])
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001167
1168 return pkg_list
1169
1170 def generateTargetsTree(self, klass=None, pkgs=None):
1171 """
1172 Generate a dependency tree of buildable targets
1173 Generate an event with the result
1174 """
1175 # if the caller hasn't specified a pkgs list default to universe
1176 if not pkgs:
1177 pkgs = ['universe']
1178 # if inherited_class passed ensure all recipes which inherit the
1179 # specified class are included in pkgs
1180 if klass:
1181 extra_pkgs = self.findInheritsClass(klass)
1182 pkgs = pkgs + extra_pkgs
1183
1184 # generate a dependency tree for all our packages
1185 tree = self.generatePkgDepTreeData(pkgs, 'build')
1186 bb.event.fire(bb.event.TargetsTreeGenerated(tree), self.data)
1187
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001188 def interactiveMode( self ):
1189 """Drop off into a shell"""
1190 try:
1191 from bb import shell
1192 except ImportError:
1193 parselog.exception("Interactive mode not available")
Andrew Geisslerc9f78652020-09-18 14:11:35 -05001194 raise bb.BBHandledException()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001195 else:
1196 shell.start( self )
1197
1198
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001199 def handleCollections(self, collections):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001200 """Handle collections"""
1201 errors = False
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001202 self.bbfile_config_priorities = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001203 if collections:
1204 collection_priorities = {}
1205 collection_depends = {}
1206 collection_list = collections.split()
1207 min_prio = 0
1208 for c in collection_list:
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001209 bb.debug(1,'Processing %s in collection list' % (c))
1210
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001211 # Get collection priority if defined explicitly
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001212 priority = self.data.getVar("BBFILE_PRIORITY_%s" % c)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001213 if priority:
1214 try:
1215 prio = int(priority)
1216 except ValueError:
1217 parselog.error("invalid value for BBFILE_PRIORITY_%s: \"%s\"", c, priority)
1218 errors = True
1219 if min_prio == 0 or prio < min_prio:
1220 min_prio = prio
1221 collection_priorities[c] = prio
1222 else:
1223 collection_priorities[c] = None
1224
1225 # Check dependencies and store information for priority calculation
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001226 deps = self.data.getVar("LAYERDEPENDS_%s" % c)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001227 if deps:
1228 try:
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001229 depDict = bb.utils.explode_dep_versions2(deps)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001230 except bb.utils.VersionStringException as vse:
1231 bb.fatal('Error parsing LAYERDEPENDS_%s: %s' % (c, str(vse)))
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001232 for dep, oplist in list(depDict.items()):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001233 if dep in collection_list:
1234 for opstr in oplist:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001235 layerver = self.data.getVar("LAYERVERSION_%s" % dep)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001236 (op, depver) = opstr.split()
1237 if layerver:
1238 try:
1239 res = bb.utils.vercmp_string_op(layerver, depver, op)
1240 except bb.utils.VersionStringException as vse:
1241 bb.fatal('Error parsing LAYERDEPENDS_%s: %s' % (c, str(vse)))
1242 if not res:
1243 parselog.error("Layer '%s' depends on version %s of layer '%s', but version %s is currently enabled in your configuration. Check that you are using the correct matching versions/branches of these two layers.", c, opstr, dep, layerver)
1244 errors = True
1245 else:
1246 parselog.error("Layer '%s' depends on version %s of layer '%s', which exists in your configuration but does not specify a version. Check that you are using the correct matching versions/branches of these two layers.", c, opstr, dep)
1247 errors = True
1248 else:
1249 parselog.error("Layer '%s' depends on layer '%s', but this layer is not enabled in your configuration", c, dep)
1250 errors = True
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001251 collection_depends[c] = list(depDict.keys())
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001252 else:
1253 collection_depends[c] = []
1254
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001255 # Check recommends and store information for priority calculation
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001256 recs = self.data.getVar("LAYERRECOMMENDS_%s" % c)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001257 if recs:
1258 try:
1259 recDict = bb.utils.explode_dep_versions2(recs)
1260 except bb.utils.VersionStringException as vse:
1261 bb.fatal('Error parsing LAYERRECOMMENDS_%s: %s' % (c, str(vse)))
1262 for rec, oplist in list(recDict.items()):
1263 if rec in collection_list:
1264 if oplist:
1265 opstr = oplist[0]
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001266 layerver = self.data.getVar("LAYERVERSION_%s" % rec)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001267 if layerver:
1268 (op, recver) = opstr.split()
1269 try:
1270 res = bb.utils.vercmp_string_op(layerver, recver, op)
1271 except bb.utils.VersionStringException as vse:
1272 bb.fatal('Error parsing LAYERRECOMMENDS_%s: %s' % (c, str(vse)))
1273 if not res:
Andrew Geissler87f5cff2022-09-30 13:13:31 -05001274 parselog.debug3("Layer '%s' recommends version %s of layer '%s', but version %s is currently enabled in your configuration. Check that you are using the correct matching versions/branches of these two layers.", c, opstr, rec, layerver)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001275 continue
1276 else:
Andrew Geissler87f5cff2022-09-30 13:13:31 -05001277 parselog.debug3("Layer '%s' recommends version %s of layer '%s', which exists in your configuration but does not specify a version. Check that you are using the correct matching versions/branches of these two layers.", c, opstr, rec)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001278 continue
Andrew Geissler87f5cff2022-09-30 13:13:31 -05001279 parselog.debug3("Layer '%s' recommends layer '%s', so we are adding it", c, rec)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001280 collection_depends[c].append(rec)
1281 else:
Andrew Geissler87f5cff2022-09-30 13:13:31 -05001282 parselog.debug3("Layer '%s' recommends layer '%s', but this layer is not enabled in your configuration", c, rec)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001283
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001284 # Recursively work out collection priorities based on dependencies
1285 def calc_layer_priority(collection):
1286 if not collection_priorities[collection]:
1287 max_depprio = min_prio
1288 for dep in collection_depends[collection]:
1289 calc_layer_priority(dep)
1290 depprio = collection_priorities[dep]
1291 if depprio > max_depprio:
1292 max_depprio = depprio
1293 max_depprio += 1
Andrew Geissler87f5cff2022-09-30 13:13:31 -05001294 parselog.debug("Calculated priority of layer %s as %d", collection, max_depprio)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001295 collection_priorities[collection] = max_depprio
1296
1297 # Calculate all layer priorities using calc_layer_priority and store in bbfile_config_priorities
1298 for c in collection_list:
1299 calc_layer_priority(c)
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001300 regex = self.data.getVar("BBFILE_PATTERN_%s" % c)
Andrew Geissler82c905d2020-04-13 13:39:40 -05001301 if regex is None:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001302 parselog.error("BBFILE_PATTERN_%s not defined" % c)
1303 errors = True
1304 continue
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001305 elif regex == "":
Andrew Geissler87f5cff2022-09-30 13:13:31 -05001306 parselog.debug("BBFILE_PATTERN_%s is empty" % c)
Brad Bishop19323692019-04-05 15:28:33 -04001307 cre = re.compile('^NULL$')
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001308 errors = False
1309 else:
1310 try:
1311 cre = re.compile(regex)
1312 except re.error:
1313 parselog.error("BBFILE_PATTERN_%s \"%s\" is not a valid regular expression", c, regex)
1314 errors = True
1315 continue
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001316 self.bbfile_config_priorities.append((c, regex, cre, collection_priorities[c]))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001317 if errors:
1318 # We've already printed the actual error(s)
1319 raise CollectionError("Errors during parsing layer configuration")
1320
1321 def buildSetVars(self):
1322 """
1323 Setup any variables needed before starting a build
1324 """
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001325 t = time.gmtime()
1326 for mc in self.databuilder.mcdata:
1327 ds = self.databuilder.mcdata[mc]
1328 if not ds.getVar("BUILDNAME", False):
1329 ds.setVar("BUILDNAME", "${DATE}${TIME}")
1330 ds.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S', t))
1331 ds.setVar("DATE", time.strftime('%Y%m%d', t))
1332 ds.setVar("TIME", time.strftime('%H%M%S', t))
1333
1334 def reset_mtime_caches(self):
1335 """
1336 Reset mtime caches - this is particularly important when memory resident as something
1337 which is cached is not unlikely to have changed since the last invocation (e.g. a
1338 file associated with a recipe might have been modified by the user).
1339 """
1340 build.reset_cache()
1341 bb.fetch._checksum_cache.mtime_cache.clear()
1342 siggen_cache = getattr(bb.parse.siggen, 'checksum_cache', None)
1343 if siggen_cache:
1344 bb.parse.siggen.checksum_cache.mtime_cache.clear()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001345
Andrew Geissler5a43b432020-06-13 10:46:56 -05001346 def matchFiles(self, bf, mc=''):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001347 """
1348 Find the .bb files which match the expression in 'buildfile'.
1349 """
1350 if bf.startswith("/") or bf.startswith("../"):
1351 bf = os.path.abspath(bf)
1352
Andrew Geissler5a43b432020-06-13 10:46:56 -05001353 self.collections = {mc: CookerCollectFiles(self.bbfile_config_priorities, mc)}
1354 filelist, masked, searchdirs = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc])
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001355 try:
1356 os.stat(bf)
1357 bf = os.path.abspath(bf)
1358 return [bf]
1359 except OSError:
1360 regexp = re.compile(bf)
1361 matches = []
1362 for f in filelist:
1363 if regexp.search(f) and os.path.isfile(f):
1364 matches.append(f)
1365 return matches
1366
Andrew Geissler5a43b432020-06-13 10:46:56 -05001367 def matchFile(self, buildfile, mc=''):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001368 """
1369 Find the .bb file which matches the expression in 'buildfile'.
1370 Raise an error if multiple files
1371 """
Andrew Geissler5a43b432020-06-13 10:46:56 -05001372 matches = self.matchFiles(buildfile, mc)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001373 if len(matches) != 1:
1374 if matches:
1375 msg = "Unable to match '%s' to a specific recipe file - %s matches found:" % (buildfile, len(matches))
1376 if matches:
1377 for f in matches:
1378 msg += "\n %s" % f
1379 parselog.error(msg)
1380 else:
1381 parselog.error("Unable to find any recipe file matching '%s'" % buildfile)
1382 raise NoSpecificMatch
1383 return matches[0]
1384
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001385 def buildFile(self, buildfile, task):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001386 """
1387 Build the file matching regexp buildfile
1388 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001389 bb.event.fire(bb.event.BuildInit(), self.data)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001390
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001391 # Too many people use -b because they think it's how you normally
1392 # specify a target to be built, so show a warning
1393 bb.warn("Buildfile specified, dependencies will not be handled. If this is not what you want, do not use -b / --buildfile.")
1394
1395 self.buildFileInternal(buildfile, task)
1396
1397 def buildFileInternal(self, buildfile, task, fireevents=True, quietlog=False):
1398 """
1399 Build the file matching regexp buildfile
1400 """
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001401
1402 # Parse the configuration here. We need to do it explicitly here since
1403 # buildFile() doesn't use the cache
1404 self.parseConfiguration()
1405
1406 # If we are told to do the None task then query the default task
Andrew Geissler82c905d2020-04-13 13:39:40 -05001407 if task is None:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001408 task = self.configuration.cmd
Brad Bishop37a0e4d2017-12-04 01:01:44 -05001409 if not task.startswith("do_"):
1410 task = "do_%s" % task
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001411
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001412 fn, cls, mc = bb.cache.virtualfn2realfn(buildfile)
Andrew Geissler5a43b432020-06-13 10:46:56 -05001413 fn = self.matchFile(fn, mc)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001414
1415 self.buildSetVars()
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001416 self.reset_mtime_caches()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001417
Andrew Geissler5a43b432020-06-13 10:46:56 -05001418 bb_caches = bb.cache.MulticonfigCache(self.databuilder, self.data_hash, self.caches_array)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001419
Andrew Geissler5a43b432020-06-13 10:46:56 -05001420 infos = bb_caches[mc].parse(fn, self.collections[mc].get_file_appends(fn))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001421 infos = dict(infos)
1422
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001423 fn = bb.cache.realfn2virtual(fn, cls, mc)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001424 try:
1425 info_array = infos[fn]
1426 except KeyError:
1427 bb.fatal("%s does not exist" % fn)
1428
1429 if info_array[0].skipped:
1430 bb.fatal("%s was skipped: %s" % (fn, info_array[0].skipreason))
1431
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001432 self.recipecaches[mc].add_from_recipeinfo(fn, info_array)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001433
1434 # Tweak some variables
1435 item = info_array[0].pn
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001436 self.recipecaches[mc].ignored_dependencies = set()
1437 self.recipecaches[mc].bbfile_priority[fn] = 1
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001438 self.configuration.limited_deps = True
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001439
1440 # Remove external dependencies
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001441 self.recipecaches[mc].task_deps[fn]['depends'] = {}
1442 self.recipecaches[mc].deps[fn] = []
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001443 self.recipecaches[mc].rundeps[fn] = defaultdict(list)
1444 self.recipecaches[mc].runrecs[fn] = defaultdict(list)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001445
Andrew Geissler517393d2023-01-13 08:55:19 -06001446 bb.parse.siggen.setup_datacache(self.recipecaches)
1447
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001448 # Invalidate task for target if force mode active
1449 if self.configuration.force:
1450 logger.verbose("Invalidate task %s, %s", task, fn)
Andrew Geissler517393d2023-01-13 08:55:19 -06001451 bb.parse.siggen.invalidate_task(task, fn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001452
1453 # Setup taskdata structure
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001454 taskdata = {}
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001455 taskdata[mc] = bb.taskdata.TaskData(self.configuration.halt)
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001456 taskdata[mc].add_provider(self.databuilder.mcdata[mc], self.recipecaches[mc], item)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001457
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001458 if quietlog:
1459 rqloglevel = bb.runqueue.logger.getEffectiveLevel()
1460 bb.runqueue.logger.setLevel(logging.WARNING)
1461
1462 buildname = self.databuilder.mcdata[mc].getVar("BUILDNAME")
1463 if fireevents:
1464 bb.event.fire(bb.event.BuildStarted(buildname, [item]), self.databuilder.mcdata[mc])
Andrew Geissler517393d2023-01-13 08:55:19 -06001465 bb.event.enable_heartbeat()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001466
1467 # Execute the runqueue
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001468 runlist = [[mc, item, task, fn]]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001469
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001470 rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001471
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001472 def buildFileIdle(server, rq, halt):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001473
1474 msg = None
1475 interrupted = 0
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001476 if halt or self.state == state.forceshutdown:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001477 rq.finish_runqueue(True)
1478 msg = "Forced shutdown"
1479 interrupted = 2
1480 elif self.state == state.shutdown:
1481 rq.finish_runqueue(False)
1482 msg = "Stopped build"
1483 interrupted = 1
1484 failures = 0
1485 try:
1486 retval = rq.execute_runqueue()
1487 except runqueue.TaskFailure as exc:
1488 failures += len(exc.args)
1489 retval = False
1490 except SystemExit as exc:
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001491 if quietlog:
1492 bb.runqueue.logger.setLevel(rqloglevel)
Andrew Geissler517393d2023-01-13 08:55:19 -06001493 return bb.server.process.idleFinish(str(exc))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001494
1495 if not retval:
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001496 if fireevents:
1497 bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runtaskentries), buildname, item, failures, interrupted), self.databuilder.mcdata[mc])
Andrew Geissler517393d2023-01-13 08:55:19 -06001498 bb.event.disable_heartbeat()
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001499 # We trashed self.recipecaches above
1500 self.parsecache_valid = False
1501 self.configuration.limited_deps = False
1502 bb.parse.siggen.reset(self.data)
1503 if quietlog:
1504 bb.runqueue.logger.setLevel(rqloglevel)
Andrew Geissler517393d2023-01-13 08:55:19 -06001505 return bb.server.process.idleFinish(msg)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001506 if retval is True:
1507 return True
1508 return retval
1509
Andrew Geissler635e0e42020-08-21 15:58:33 -05001510 self.idleCallBackRegister(buildFileIdle, rq)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001511
1512 def buildTargets(self, targets, task):
1513 """
1514 Attempt to build the targets specified
1515 """
1516
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001517 def buildTargetsIdle(server, rq, halt):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001518 msg = None
1519 interrupted = 0
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001520 if halt or self.state == state.forceshutdown:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001521 rq.finish_runqueue(True)
1522 msg = "Forced shutdown"
1523 interrupted = 2
1524 elif self.state == state.shutdown:
1525 rq.finish_runqueue(False)
1526 msg = "Stopped build"
1527 interrupted = 1
1528 failures = 0
1529 try:
1530 retval = rq.execute_runqueue()
1531 except runqueue.TaskFailure as exc:
1532 failures += len(exc.args)
1533 retval = False
1534 except SystemExit as exc:
Andrew Geissler517393d2023-01-13 08:55:19 -06001535 return bb.server.process.idleFinish(str(exc))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001536
1537 if not retval:
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001538 try:
1539 for mc in self.multiconfigs:
1540 bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runtaskentries), buildname, targets, failures, interrupted), self.databuilder.mcdata[mc])
1541 finally:
Andrew Geissler517393d2023-01-13 08:55:19 -06001542 bb.event.disable_heartbeat()
1543 return bb.server.process.idleFinish(msg)
1544
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001545 if retval is True:
1546 return True
1547 return retval
1548
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001549 self.reset_mtime_caches()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001550 self.buildSetVars()
1551
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001552 # If we are told to do the None task then query the default task
Andrew Geissler82c905d2020-04-13 13:39:40 -05001553 if task is None:
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001554 task = self.configuration.cmd
1555
1556 if not task.startswith("do_"):
1557 task = "do_%s" % task
1558
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001559 packages = [target if ':' in target else '%s:%s' % (target, task) for target in targets]
1560
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001561 bb.event.fire(bb.event.BuildInit(packages), self.data)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001562
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001563 taskdata, runlist = self.buildTaskData(targets, task, self.configuration.halt)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001564
1565 buildname = self.data.getVar("BUILDNAME", False)
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001566
1567 # make targets to always look as <target>:do_<task>
1568 ntargets = []
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001569 for target in runlist:
1570 if target[0]:
Brad Bishop15ae2502019-06-18 21:44:24 -04001571 ntargets.append("mc:%s:%s:%s" % (target[0], target[1], target[2]))
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001572 ntargets.append("%s:%s" % (target[1], target[2]))
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001573
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001574 for mc in self.multiconfigs:
1575 bb.event.fire(bb.event.BuildStarted(buildname, ntargets), self.databuilder.mcdata[mc])
Andrew Geissler517393d2023-01-13 08:55:19 -06001576 bb.event.enable_heartbeat()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001577
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001578 rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001579 if 'universe' in targets:
1580 rq.rqdata.warn_multi_bb = True
1581
Andrew Geissler635e0e42020-08-21 15:58:33 -05001582 self.idleCallBackRegister(buildTargetsIdle, rq)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001583
1584
1585 def getAllKeysWithFlags(self, flaglist):
1586 dump = {}
1587 for k in self.data.keys():
1588 try:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001589 expand = True
1590 flags = self.data.getVarFlags(k)
1591 if flags and "func" in flags and "python" in flags:
1592 expand = False
1593 v = self.data.getVar(k, expand)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001594 if not k.startswith("__") and not isinstance(v, bb.data_smart.DataSmart):
1595 dump[k] = {
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001596 'v' : str(v) ,
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001597 'history' : self.data.varhistory.variable(k),
1598 }
1599 for d in flaglist:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001600 if flags and d in flags:
1601 dump[k][d] = flags[d]
1602 else:
1603 dump[k][d] = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001604 except Exception as e:
1605 print(e)
1606 return dump
1607
1608
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001609 def updateCacheSync(self):
1610 if self.state == state.running:
1611 return
1612
1613 # reload files for which we got notifications
1614 for p in self.inotify_modified_files:
1615 bb.parse.update_cache(p)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001616 if p in bb.parse.BBHandler.cached_statements:
1617 del bb.parse.BBHandler.cached_statements[p]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001618 self.inotify_modified_files = []
1619
1620 if not self.baseconfig_valid:
Andrew Geisslerd1e89492021-02-12 15:35:20 -06001621 logger.debug("Reloading base configuration data")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001622 self.initConfigurationData()
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001623 self.handlePRServ()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001624
1625 # This is called for all async commands when self.state != running
1626 def updateCache(self):
1627 if self.state == state.running:
1628 return
1629
1630 if self.state in (state.shutdown, state.forceshutdown, state.error):
1631 if hasattr(self.parser, 'shutdown'):
Andrew Geissler9aee5002022-03-30 16:27:02 +00001632 self.parser.shutdown(clean=False)
Andrew Geisslerc9f78652020-09-18 14:11:35 -05001633 self.parser.final_cleanup()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001634 raise bb.BBHandledException()
1635
1636 if self.state != state.parsing:
1637 self.updateCacheSync()
1638
1639 if self.state != state.parsing and not self.parsecache_valid:
Patrick Williamsde0582f2022-04-08 10:23:27 -05001640 self.setupParserWatcher()
1641
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001642 bb.parse.siggen.reset(self.data)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001643 self.parseConfiguration ()
1644 if CookerFeatures.SEND_SANITYEVENTS in self.featureset:
Brad Bishop37a0e4d2017-12-04 01:01:44 -05001645 for mc in self.multiconfigs:
1646 bb.event.fire(bb.event.SanityCheck(False), self.databuilder.mcdata[mc])
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001647
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001648 for mc in self.multiconfigs:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001649 ignore = self.databuilder.mcdata[mc].getVar("ASSUME_PROVIDED") or ""
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001650 self.recipecaches[mc].ignored_dependencies = set(ignore.split())
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001651
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001652 for dep in self.configuration.extra_assume_provided:
1653 self.recipecaches[mc].ignored_dependencies.add(dep)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001654
Andrew Geissler5a43b432020-06-13 10:46:56 -05001655 self.collections = {}
1656
1657 mcfilelist = {}
1658 total_masked = 0
1659 searchdirs = set()
1660 for mc in self.multiconfigs:
1661 self.collections[mc] = CookerCollectFiles(self.bbfile_config_priorities, mc)
1662 (filelist, masked, search) = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc])
1663
1664 mcfilelist[mc] = filelist
1665 total_masked += masked
1666 searchdirs |= set(search)
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001667
1668 # Add inotify watches for directories searched for bb/bbappend files
1669 for dirent in searchdirs:
1670 self.add_filewatch([[dirent]], dirs=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001671
Andrew Geissler5a43b432020-06-13 10:46:56 -05001672 self.parser = CookerParser(self, mcfilelist, total_masked)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001673 self.parsecache_valid = True
1674
1675 self.state = state.parsing
1676
1677 if not self.parser.parse_next():
Andrew Geissler87f5cff2022-09-30 13:13:31 -05001678 collectlog.debug("parsing complete")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001679 if self.parser.error:
1680 raise bb.BBHandledException()
1681 self.show_appends_with_no_recipes()
1682 self.handlePrefProviders()
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001683 for mc in self.multiconfigs:
Andrew Geisslerb7d28612020-07-24 16:15:54 -05001684 self.recipecaches[mc].bbfile_priority = self.collections[mc].collection_priorities(self.recipecaches[mc].pkg_fn, self.parser.mcfilelist[mc], self.data)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001685 self.state = state.running
1686
1687 # Send an event listing all stamps reachable after parsing
1688 # which the metadata may use to clean up stale data
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001689 for mc in self.multiconfigs:
1690 event = bb.event.ReachableStamps(self.recipecaches[mc].stamp)
1691 bb.event.fire(event, self.databuilder.mcdata[mc])
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001692 return None
1693
1694 return True
1695
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001696 def checkPackages(self, pkgs_to_build, task=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001697
1698 # Return a copy, don't modify the original
1699 pkgs_to_build = pkgs_to_build[:]
1700
Andrew Geissler595f6302022-01-24 19:11:47 +00001701 if not pkgs_to_build:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001702 raise NothingToBuild
1703
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001704 ignore = (self.data.getVar("ASSUME_PROVIDED") or "").split()
Andrew Geisslerb7d28612020-07-24 16:15:54 -05001705 for pkg in pkgs_to_build.copy():
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001706 if pkg in ignore:
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001707 parselog.warning("Explicit target \"%s\" is in ASSUME_PROVIDED, ignoring" % pkg)
Brad Bishop15ae2502019-06-18 21:44:24 -04001708 if pkg.startswith("multiconfig:"):
1709 pkgs_to_build.remove(pkg)
1710 pkgs_to_build.append(pkg.replace("multiconfig:", "mc:"))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001711
1712 if 'world' in pkgs_to_build:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001713 pkgs_to_build.remove('world')
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001714 for mc in self.multiconfigs:
1715 bb.providers.buildWorldTargetList(self.recipecaches[mc], task)
1716 for t in self.recipecaches[mc].world_target:
1717 if mc:
Brad Bishop15ae2502019-06-18 21:44:24 -04001718 t = "mc:" + mc + ":" + t
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001719 pkgs_to_build.append(t)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001720
1721 if 'universe' in pkgs_to_build:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001722 parselog.verbnote("The \"universe\" target is only intended for testing and may produce errors.")
Andrew Geissler87f5cff2022-09-30 13:13:31 -05001723 parselog.debug("collating packages for \"universe\"")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001724 pkgs_to_build.remove('universe')
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001725 for mc in self.multiconfigs:
1726 for t in self.recipecaches[mc].universe_target:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001727 if task:
1728 foundtask = False
1729 for provider_fn in self.recipecaches[mc].providers[t]:
1730 if task in self.recipecaches[mc].task_deps[provider_fn]['tasks']:
1731 foundtask = True
1732 break
1733 if not foundtask:
1734 bb.debug(1, "Skipping %s for universe tasks as task %s doesn't exist" % (t, task))
1735 continue
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001736 if mc:
Brad Bishop15ae2502019-06-18 21:44:24 -04001737 t = "mc:" + mc + ":" + t
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001738 pkgs_to_build.append(t)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001739
1740 return pkgs_to_build
1741
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001742 def pre_serve(self):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001743 return
1744
1745 def post_serve(self):
Andrew Geisslerc9f78652020-09-18 14:11:35 -05001746 self.shutdown(force=True)
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001747 prserv.serv.auto_shutdown()
Patrick Williams45852732022-04-02 08:58:32 -05001748 if hasattr(bb.parse, "siggen"):
1749 bb.parse.siggen.exit()
Brad Bishop08902b02019-08-20 09:16:51 -04001750 if self.hashserv:
1751 self.hashserv.process.terminate()
1752 self.hashserv.process.join()
Andrew Geisslerc3d88e42020-10-02 09:45:00 -05001753 if hasattr(self, "data"):
1754 bb.event.fire(CookerExit(), self.data)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001755
Andrew Geissler517393d2023-01-13 08:55:19 -06001756 def shutdown(self, force=False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001757 if force:
1758 self.state = state.forceshutdown
1759 else:
1760 self.state = state.shutdown
1761
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001762 if self.parser:
Andrew Geissler517393d2023-01-13 08:55:19 -06001763 self.parser.shutdown(clean=False)
Andrew Geisslerc9f78652020-09-18 14:11:35 -05001764 self.parser.final_cleanup()
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001765
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001766 def finishcommand(self):
Andrew Geissler517393d2023-01-13 08:55:19 -06001767 if hasattr(self.parser, 'shutdown'):
1768 self.parser.shutdown(clean=False)
1769 self.parser.final_cleanup()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001770 self.state = state.initial
1771
1772 def reset(self):
Patrick Williams45852732022-04-02 08:58:32 -05001773 if hasattr(bb.parse, "siggen"):
1774 bb.parse.siggen.exit()
Andrew Geissler517393d2023-01-13 08:55:19 -06001775 self.finishcommand()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001776 self.initConfigurationData()
Brad Bishop08902b02019-08-20 09:16:51 -04001777 self.handlePRServ()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001778
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001779 def clientComplete(self):
1780 """Called when the client is done using the server"""
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001781 self.finishcommand()
1782 self.extraconfigdata = {}
1783 self.command.reset()
Andrew Geisslerc9f78652020-09-18 14:11:35 -05001784 if hasattr(self, "data"):
1785 self.databuilder.reset()
1786 self.data = self.databuilder.data
Andrew Geissler517393d2023-01-13 08:55:19 -06001787 # In theory tinfoil could have modified the base data before parsing,
1788 # ideally need to track if anything did modify the datastore
Andrew Geissler82c905d2020-04-13 13:39:40 -05001789 self.parsecache_valid = False
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001790
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001791
1792class CookerExit(bb.event.Event):
1793 """
1794 Notify clients of the Cooker shutdown
1795 """
1796
1797 def __init__(self):
1798 bb.event.Event.__init__(self)
1799
1800
1801class CookerCollectFiles(object):
Andrew Geissler5a43b432020-06-13 10:46:56 -05001802 def __init__(self, priorities, mc=''):
1803 self.mc = mc
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001804 self.bbappends = []
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001805 # Priorities is a list of tuples, with the second element as the pattern.
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001806 # We need to sort the list with the longest pattern first, and so on to
1807 # the shortest. This allows nested layers to be properly evaluated.
1808 self.bbfile_config_priorities = sorted(priorities, key=lambda tup: tup[1], reverse=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001809
Andrew Geisslerb7d28612020-07-24 16:15:54 -05001810 def calc_bbfile_priority(self, filename):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001811 for _, _, regex, pri in self.bbfile_config_priorities:
1812 if regex.match(filename):
Andrew Geisslerb7d28612020-07-24 16:15:54 -05001813 return pri, regex
1814 return 0, None
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001815
1816 def get_bbfiles(self):
1817 """Get list of default .bb files by reading out the current directory"""
1818 path = os.getcwd()
1819 contents = os.listdir(path)
1820 bbfiles = []
1821 for f in contents:
1822 if f.endswith(".bb"):
1823 bbfiles.append(os.path.abspath(os.path.join(path, f)))
1824 return bbfiles
1825
1826 def find_bbfiles(self, path):
1827 """Find all the .bb and .bbappend files in a directory"""
1828 found = []
1829 for dir, dirs, files in os.walk(path):
1830 for ignored in ('SCCS', 'CVS', '.svn'):
1831 if ignored in dirs:
1832 dirs.remove(ignored)
1833 found += [os.path.join(dir, f) for f in files if (f.endswith(['.bb', '.bbappend']))]
1834
1835 return found
1836
1837 def collect_bbfiles(self, config, eventdata):
1838 """Collect all available .bb build files"""
1839 masked = 0
1840
Andrew Geissler87f5cff2022-09-30 13:13:31 -05001841 collectlog.debug("collecting .bb files")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001842
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001843 files = (config.getVar( "BBFILES") or "").split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001844
1845 # Sort files by priority
Andrew Geisslerb7d28612020-07-24 16:15:54 -05001846 files.sort( key=lambda fileitem: self.calc_bbfile_priority(fileitem)[0] )
Andrew Geisslerc9f78652020-09-18 14:11:35 -05001847 config.setVar("BBFILES_PRIORITIZED", " ".join(files))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001848
Andrew Geissler595f6302022-01-24 19:11:47 +00001849 if not files:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001850 files = self.get_bbfiles()
1851
Andrew Geissler595f6302022-01-24 19:11:47 +00001852 if not files:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001853 collectlog.error("no recipe files to build, check your BBPATH and BBFILES?")
1854 bb.event.fire(CookerExit(), eventdata)
1855
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001856 # We need to track where we look so that we can add inotify watches. There
1857 # is no nice way to do this, this is horrid. We intercept the os.listdir()
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001858 # (or os.scandir() for python 3.6+) calls while we run glob().
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001859 origlistdir = os.listdir
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001860 if hasattr(os, 'scandir'):
1861 origscandir = os.scandir
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001862 searchdirs = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001863
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001864 def ourlistdir(d):
1865 searchdirs.append(d)
1866 return origlistdir(d)
1867
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001868 def ourscandir(d):
1869 searchdirs.append(d)
1870 return origscandir(d)
1871
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001872 os.listdir = ourlistdir
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001873 if hasattr(os, 'scandir'):
1874 os.scandir = ourscandir
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001875 try:
1876 # Can't use set here as order is important
1877 newfiles = []
1878 for f in files:
1879 if os.path.isdir(f):
1880 dirfiles = self.find_bbfiles(f)
1881 for g in dirfiles:
1882 if g not in newfiles:
1883 newfiles.append(g)
1884 else:
1885 globbed = glob.glob(f)
1886 if not globbed and os.path.exists(f):
1887 globbed = [f]
1888 # glob gives files in order on disk. Sort to be deterministic.
1889 for g in sorted(globbed):
1890 if g not in newfiles:
1891 newfiles.append(g)
1892 finally:
1893 os.listdir = origlistdir
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001894 if hasattr(os, 'scandir'):
1895 os.scandir = origscandir
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001896
1897 bbmask = config.getVar('BBMASK')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001898
1899 if bbmask:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001900 # First validate the individual regular expressions and ignore any
1901 # that do not compile
1902 bbmasks = []
1903 for mask in bbmask.split():
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001904 # When constructing an older style single regex, it's possible for BBMASK
1905 # to end up beginning with '|', which matches and masks _everything_.
1906 if mask.startswith("|"):
Andrew Geissler82c905d2020-04-13 13:39:40 -05001907 collectlog.warning("BBMASK contains regular expression beginning with '|', fixing: %s" % mask)
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001908 mask = mask[1:]
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001909 try:
1910 re.compile(mask)
1911 bbmasks.append(mask)
Andrew Geissler78b72792022-06-14 06:47:25 -05001912 except re.error:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001913 collectlog.critical("BBMASK contains an invalid regular expression, ignoring: %s" % mask)
1914
1915 # Then validate the combined regular expressions. This should never
1916 # fail, but better safe than sorry...
1917 bbmask = "|".join(bbmasks)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001918 try:
1919 bbmask_compiled = re.compile(bbmask)
Andrew Geissler78b72792022-06-14 06:47:25 -05001920 except re.error:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001921 collectlog.critical("BBMASK is not a valid regular expression, ignoring: %s" % bbmask)
1922 bbmask = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001923
1924 bbfiles = []
1925 bbappend = []
1926 for f in newfiles:
1927 if bbmask and bbmask_compiled.search(f):
Andrew Geissler87f5cff2022-09-30 13:13:31 -05001928 collectlog.debug("skipping masked file %s", f)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001929 masked += 1
1930 continue
1931 if f.endswith('.bb'):
1932 bbfiles.append(f)
1933 elif f.endswith('.bbappend'):
1934 bbappend.append(f)
1935 else:
Andrew Geissler87f5cff2022-09-30 13:13:31 -05001936 collectlog.debug("skipping %s: unknown file extension", f)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001937
1938 # Build a list of .bbappend files for each .bb file
1939 for f in bbappend:
1940 base = os.path.basename(f).replace('.bbappend', '.bb')
1941 self.bbappends.append((base, f))
1942
1943 # Find overlayed recipes
1944 # bbfiles will be in priority order which makes this easy
1945 bbfile_seen = dict()
1946 self.overlayed = defaultdict(list)
1947 for f in reversed(bbfiles):
1948 base = os.path.basename(f)
1949 if base not in bbfile_seen:
1950 bbfile_seen[base] = f
1951 else:
1952 topfile = bbfile_seen[base]
1953 self.overlayed[topfile].append(f)
1954
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001955 return (bbfiles, masked, searchdirs)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001956
1957 def get_file_appends(self, fn):
1958 """
1959 Returns a list of .bbappend files to apply to fn
1960 """
1961 filelist = []
1962 f = os.path.basename(fn)
1963 for b in self.bbappends:
1964 (bbappend, filename) = b
1965 if (bbappend == f) or ('%' in bbappend and bbappend.startswith(f[:bbappend.index('%')])):
1966 filelist.append(filename)
Andrew Geissler5a43b432020-06-13 10:46:56 -05001967 return tuple(filelist)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001968
Andrew Geisslerb7d28612020-07-24 16:15:54 -05001969 def collection_priorities(self, pkgfns, fns, d):
1970 # Return the priorities of the entries in pkgfns
1971 # Also check that all the regexes in self.bbfile_config_priorities are used
1972 # (but to do that we need to ensure skipped recipes aren't counted, nor
1973 # collections in BBFILE_PATTERN_IGNORE_EMPTY)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001974
1975 priorities = {}
Andrew Geisslerb7d28612020-07-24 16:15:54 -05001976 seen = set()
1977 matched = set()
1978
1979 matched_regex = set()
1980 unmatched_regex = set()
1981 for _, _, regex, _ in self.bbfile_config_priorities:
1982 unmatched_regex.add(regex)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001983
1984 # Calculate priorities for each file
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001985 for p in pkgfns:
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001986 realfn, cls, mc = bb.cache.virtualfn2realfn(p)
Andrew Geisslerb7d28612020-07-24 16:15:54 -05001987 priorities[p], regex = self.calc_bbfile_priority(realfn)
1988 if regex in unmatched_regex:
1989 matched_regex.add(regex)
1990 unmatched_regex.remove(regex)
1991 seen.add(realfn)
1992 if regex:
1993 matched.add(realfn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001994
Andrew Geisslerb7d28612020-07-24 16:15:54 -05001995 if unmatched_regex:
1996 # Account for bbappend files
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001997 for b in self.bbappends:
1998 (bbfile, append) = b
Andrew Geisslerb7d28612020-07-24 16:15:54 -05001999 seen.add(append)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002000
Andrew Geisslerb7d28612020-07-24 16:15:54 -05002001 # Account for skipped recipes
2002 seen.update(fns)
2003
2004 seen.difference_update(matched)
2005
2006 def already_matched(fn):
2007 for regex in matched_regex:
2008 if regex.match(fn):
2009 return True
2010 return False
2011
2012 for unmatch in unmatched_regex.copy():
2013 for fn in seen:
2014 if unmatch.match(fn):
2015 # If the bbappend or file was already matched by another regex, skip it
2016 # e.g. for a layer within a layer, the outer regex could match, the inner
2017 # regex may match nothing and we should warn about that
2018 if already_matched(fn):
2019 continue
2020 unmatched_regex.remove(unmatch)
2021 break
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002022
2023 for collection, pattern, regex, _ in self.bbfile_config_priorities:
Andrew Geisslerb7d28612020-07-24 16:15:54 -05002024 if regex in unmatched_regex:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05002025 if d.getVar('BBFILE_PATTERN_IGNORE_EMPTY_%s' % collection) != '1':
Andrew Geissler5a43b432020-06-13 10:46:56 -05002026 collectlog.warning("No bb files in %s matched BBFILE_PATTERN_%s '%s'" % (self.mc if self.mc else 'default',
2027 collection, pattern))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002028
2029 return priorities
2030
2031class ParsingFailure(Exception):
2032 def __init__(self, realexception, recipe):
2033 self.realexception = realexception
2034 self.recipe = recipe
2035 Exception.__init__(self, realexception, recipe)
2036
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002037class Parser(multiprocessing.Process):
Andrew Geissler9aee5002022-03-30 16:27:02 +00002038 def __init__(self, jobs, results, quit, profile):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002039 self.jobs = jobs
2040 self.results = results
2041 self.quit = quit
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002042 multiprocessing.Process.__init__(self)
2043 self.context = bb.utils.get_context().copy()
2044 self.handlers = bb.event.get_class_handlers().copy()
2045 self.profile = profile
Andrew Geissler9aee5002022-03-30 16:27:02 +00002046 self.queue_signals = False
2047 self.signal_received = []
2048 self.signal_threadlock = threading.Lock()
2049
2050 def catch_sig(self, signum, frame):
2051 if self.queue_signals:
2052 self.signal_received.append(signum)
2053 else:
2054 self.handle_sig(signum, frame)
2055
2056 def handle_sig(self, signum, frame):
2057 if signum == signal.SIGTERM:
2058 signal.signal(signal.SIGTERM, signal.SIG_DFL)
2059 os.kill(os.getpid(), signal.SIGTERM)
2060 elif signum == signal.SIGINT:
2061 signal.default_int_handler(signum, frame)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002062
2063 def run(self):
2064
2065 if not self.profile:
2066 self.realrun()
2067 return
2068
2069 try:
2070 import cProfile as profile
2071 except:
2072 import profile
2073 prof = profile.Profile()
2074 try:
2075 profile.Profile.runcall(prof, self.realrun)
2076 finally:
2077 logfile = "profile-parse-%s.log" % multiprocessing.current_process().name
2078 prof.dump_stats(logfile)
2079
2080 def realrun(self):
Andrew Geissler9aee5002022-03-30 16:27:02 +00002081 # Signal handling here is hard. We must not terminate any process or thread holding the write
2082 # lock for the event stream as it will not be released, ever, and things will hang.
2083 # Python handles signals in the main thread/process but they can be raised from any thread and
2084 # we want to defer processing of any SIGTERM/SIGINT signal until we're outside the critical section
2085 # and don't hold the lock (see server/process.py). We therefore always catch the signals (so any
2086 # new thread should also do so) and we defer handling but we handle with the local thread lock
2087 # held (a threading lock, not a multiprocessing one) so that no other thread in the process
2088 # can be in the critical section.
2089 signal.signal(signal.SIGTERM, self.catch_sig)
2090 signal.signal(signal.SIGHUP, signal.SIG_DFL)
2091 signal.signal(signal.SIGINT, self.catch_sig)
2092 bb.utils.set_process_name(multiprocessing.current_process().name)
2093 multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, exitpriority=1)
2094 multiprocessing.util.Finalize(None, bb.fetch.fetcher_parse_save, exitpriority=1)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002095
2096 pending = []
Andrew Geissler517393d2023-01-13 08:55:19 -06002097 havejobs = True
Andrew Geissler9aee5002022-03-30 16:27:02 +00002098 try:
Andrew Geissler517393d2023-01-13 08:55:19 -06002099 while havejobs or pending:
2100 if self.quit.is_set():
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002101 break
Andrew Geissler9aee5002022-03-30 16:27:02 +00002102
Andrew Geissler517393d2023-01-13 08:55:19 -06002103 job = None
2104 try:
2105 job = self.jobs.pop()
2106 except IndexError:
2107 havejobs = False
2108 if job:
Andrew Geissler9aee5002022-03-30 16:27:02 +00002109 result = self.parse(*job)
2110 # Clear the siggen cache after parsing to control memory usage, its huge
2111 bb.parse.siggen.postparsing_clean_cache()
Andrew Geissler9aee5002022-03-30 16:27:02 +00002112 pending.append(result)
Andrew Geissler517393d2023-01-13 08:55:19 -06002113
2114 if pending:
2115 try:
2116 result = pending.pop()
2117 self.results.put(result, timeout=0.05)
2118 except queue.Full:
2119 pending.append(result)
Andrew Geissler9aee5002022-03-30 16:27:02 +00002120 finally:
2121 self.results.close()
2122 self.results.join_thread()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002123
Andrew Geissler5a43b432020-06-13 10:46:56 -05002124 def parse(self, mc, cache, filename, appends):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002125 try:
Andrew Geissler82c905d2020-04-13 13:39:40 -05002126 origfilter = bb.event.LogHandler.filter
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05002127 # Record the filename we're parsing into any events generated
2128 def parse_filter(self, record):
2129 record.taskpid = bb.event.worker_pid
2130 record.fn = filename
2131 return True
2132
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002133 # Reset our environment and handlers to the original settings
2134 bb.utils.set_context(self.context.copy())
2135 bb.event.set_class_handlers(self.handlers.copy())
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05002136 bb.event.LogHandler.filter = parse_filter
2137
Andrew Geissler5a43b432020-06-13 10:46:56 -05002138 return True, mc, cache.parse(filename, appends)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002139 except Exception as exc:
2140 tb = sys.exc_info()[2]
2141 exc.recipe = filename
2142 exc.traceback = list(bb.exceptions.extract_traceback(tb, context=3))
Andrew Geissler9aee5002022-03-30 16:27:02 +00002143 return True, None, exc
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002144 # Need to turn BaseExceptions into Exceptions here so we gracefully shutdown
2145 # and for example a worker thread doesn't just exit on its own in response to
2146 # a SystemExit event for example.
2147 except BaseException as exc:
Andrew Geissler9aee5002022-03-30 16:27:02 +00002148 return True, None, ParsingFailure(exc, filename)
Andrew Geissler82c905d2020-04-13 13:39:40 -05002149 finally:
2150 bb.event.LogHandler.filter = origfilter
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002151
2152class CookerParser(object):
Andrew Geissler5a43b432020-06-13 10:46:56 -05002153 def __init__(self, cooker, mcfilelist, masked):
2154 self.mcfilelist = mcfilelist
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002155 self.cooker = cooker
2156 self.cfgdata = cooker.data
2157 self.cfghash = cooker.data_hash
Patrick Williamsc0f7c042017-02-23 20:41:17 -06002158 self.cfgbuilder = cooker.databuilder
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002159
2160 # Accounting statistics
2161 self.parsed = 0
2162 self.cached = 0
2163 self.error = 0
2164 self.masked = masked
2165
2166 self.skipped = 0
2167 self.virtuals = 0
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002168
2169 self.current = 0
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002170 self.process_names = []
2171
Andrew Geissler5a43b432020-06-13 10:46:56 -05002172 self.bb_caches = bb.cache.MulticonfigCache(self.cfgbuilder, self.cfghash, cooker.caches_array)
2173 self.fromcache = set()
2174 self.willparse = set()
2175 for mc in self.cooker.multiconfigs:
2176 for filename in self.mcfilelist[mc]:
2177 appends = self.cooker.collections[mc].get_file_appends(filename)
2178 if not self.bb_caches[mc].cacheValid(filename, appends):
2179 self.willparse.add((mc, self.bb_caches[mc], filename, appends))
2180 else:
2181 self.fromcache.add((mc, self.bb_caches[mc], filename, appends))
2182
2183 self.total = len(self.fromcache) + len(self.willparse)
2184 self.toparse = len(self.willparse)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06002185 self.progress_chunk = int(max(self.toparse / 100, 1))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002186
Brad Bishop6e60e8b2018-02-01 10:27:11 -05002187 self.num_processes = min(int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS") or
Andrew Geissler5a43b432020-06-13 10:46:56 -05002188 multiprocessing.cpu_count()), self.toparse)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05002189
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002190 self.start()
2191 self.haveshutdown = False
Andrew Geisslerc9f78652020-09-18 14:11:35 -05002192 self.syncthread = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002193
Andrew Geissler517393d2023-01-13 08:55:19 -06002194 bb.cache.SiggenRecipeInfo.reset()
2195
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002196 def start(self):
2197 self.results = self.load_cached()
2198 self.processes = []
2199 if self.toparse:
2200 bb.event.fire(bb.event.ParseStarted(self.toparse), self.cfgdata)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002201
Andrew Geissler517393d2023-01-13 08:55:19 -06002202 self.parser_quit = multiprocessing.Event()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002203 self.result_queue = multiprocessing.Queue()
Brad Bishop19323692019-04-05 15:28:33 -04002204
2205 def chunkify(lst,n):
2206 return [lst[i::n] for i in range(n)]
Andrew Geissler5a43b432020-06-13 10:46:56 -05002207 self.jobs = chunkify(list(self.willparse), self.num_processes)
Brad Bishop19323692019-04-05 15:28:33 -04002208
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002209 for i in range(0, self.num_processes):
Andrew Geissler9aee5002022-03-30 16:27:02 +00002210 parser = Parser(self.jobs[i], self.result_queue, self.parser_quit, self.cooker.configuration.profile)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002211 parser.start()
2212 self.process_names.append(parser.name)
2213 self.processes.append(parser)
2214
2215 self.results = itertools.chain(self.results, self.parse_generator())
2216
Andrew Geissler9aee5002022-03-30 16:27:02 +00002217 def shutdown(self, clean=True):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002218 if not self.toparse:
2219 return
2220 if self.haveshutdown:
2221 return
2222 self.haveshutdown = True
2223
2224 if clean:
2225 event = bb.event.ParseCompleted(self.cached, self.parsed,
2226 self.skipped, self.masked,
2227 self.virtuals, self.error,
2228 self.total)
2229
2230 bb.event.fire(event, self.cfgdata)
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00002231 else:
Andrew Geissler9aee5002022-03-30 16:27:02 +00002232 bb.error("Parsing halted due to errors, see error messages above")
Andrew Geisslerc9f78652020-09-18 14:11:35 -05002233
Andrew Geissler517393d2023-01-13 08:55:19 -06002234 def sync_caches():
2235 for c in self.bb_caches.values():
2236 bb.cache.SiggenRecipeInfo.reset()
2237 c.sync()
2238
2239 self.syncthread = threading.Thread(target=sync_caches, name="SyncThread")
2240 self.syncthread.start()
2241
2242 self.parser_quit.set()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002243
Brad Bishop08902b02019-08-20 09:16:51 -04002244 # Cleanup the queue before call process.join(), otherwise there might be
2245 # deadlocks.
2246 while True:
2247 try:
2248 self.result_queue.get(timeout=0.25)
2249 except queue.Empty:
2250 break
2251
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002252 for process in self.processes:
Andrew Geissler9aee5002022-03-30 16:27:02 +00002253 process.join(0.5)
2254
2255 for process in self.processes:
2256 if process.exitcode is None:
2257 os.kill(process.pid, signal.SIGINT)
2258
2259 for process in self.processes:
2260 process.join(0.5)
2261
2262 for process in self.processes:
2263 if process.exitcode is None:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002264 process.terminate()
Andrew Geissler9aee5002022-03-30 16:27:02 +00002265
2266 for process in self.processes:
2267 process.join()
2268 # Added in 3.7, cleans up zombies
2269 if hasattr(process, "close"):
2270 process.close()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002271
Andrew Geisslerc9f78652020-09-18 14:11:35 -05002272
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05002273 bb.codeparser.parser_cache_savemerge()
Andrew Geissler517393d2023-01-13 08:55:19 -06002274 bb.cache.SiggenRecipeInfo.reset()
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05002275 bb.fetch.fetcher_parse_done()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002276 if self.cooker.configuration.profile:
2277 profiles = []
2278 for i in self.process_names:
2279 logfile = "profile-parse-%s.log" % i
2280 if os.path.exists(logfile):
2281 profiles.append(logfile)
2282
2283 pout = "profile-parse.log.processed"
2284 bb.utils.process_profilelog(profiles, pout = pout)
2285 print("Processed parsing statistics saved to %s" % (pout))
2286
Andrew Geisslerc9f78652020-09-18 14:11:35 -05002287 def final_cleanup(self):
2288 if self.syncthread:
2289 self.syncthread.join()
2290
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002291 def load_cached(self):
Andrew Geissler5a43b432020-06-13 10:46:56 -05002292 for mc, cache, filename, appends in self.fromcache:
Andrew Geissler517393d2023-01-13 08:55:19 -06002293 infos = cache.loadCached(filename, appends)
2294 yield False, mc, infos
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002295
2296 def parse_generator(self):
Andrew Geissler595f6302022-01-24 19:11:47 +00002297 empty = False
2298 while self.processes or not empty:
2299 for process in self.processes.copy():
2300 if not process.is_alive():
2301 process.join()
2302 self.processes.remove(process)
2303
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002304 if self.parsed >= self.toparse:
2305 break
2306
2307 try:
2308 result = self.result_queue.get(timeout=0.25)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06002309 except queue.Empty:
Andrew Geissler595f6302022-01-24 19:11:47 +00002310 empty = True
Andrew Geissler9aee5002022-03-30 16:27:02 +00002311 yield None, None, None
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002312 else:
Andrew Geissler595f6302022-01-24 19:11:47 +00002313 empty = False
Andrew Geissler9aee5002022-03-30 16:27:02 +00002314 yield result
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002315
Andrew Geissler595f6302022-01-24 19:11:47 +00002316 if not (self.parsed >= self.toparse):
2317 raise bb.parse.ParseError("Not all recipes parsed, parser thread killed/died? Exiting.", None)
2318
2319
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002320 def parse_next(self):
2321 result = []
2322 parsed = None
2323 try:
Andrew Geissler5a43b432020-06-13 10:46:56 -05002324 parsed, mc, result = next(self.results)
Andrew Geissler9aee5002022-03-30 16:27:02 +00002325 if isinstance(result, BaseException):
2326 # Turn exceptions back into exceptions
2327 raise result
2328 if parsed is None:
2329 # Timeout, loop back through the main loop
2330 return True
2331
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002332 except StopIteration:
2333 self.shutdown()
2334 return False
2335 except bb.BBHandledException as exc:
2336 self.error += 1
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00002337 logger.debug('Failed to parse recipe: %s' % exc.recipe)
Andrew Geissler9aee5002022-03-30 16:27:02 +00002338 self.shutdown(clean=False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002339 return False
2340 except ParsingFailure as exc:
2341 self.error += 1
2342 logger.error('Unable to parse %s: %s' %
2343 (exc.recipe, bb.exceptions.to_string(exc.realexception)))
Andrew Geissler9aee5002022-03-30 16:27:02 +00002344 self.shutdown(clean=False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002345 return False
2346 except bb.parse.ParseError as exc:
2347 self.error += 1
2348 logger.error(str(exc))
Andrew Geissler9aee5002022-03-30 16:27:02 +00002349 self.shutdown(clean=False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002350 return False
2351 except bb.data_smart.ExpansionError as exc:
2352 self.error += 1
Patrick Williamsc0f7c042017-02-23 20:41:17 -06002353 bbdir = os.path.dirname(__file__) + os.sep
2354 etype, value, _ = sys.exc_info()
2355 tb = list(itertools.dropwhile(lambda e: e.filename.startswith(bbdir), exc.traceback))
2356 logger.error('ExpansionError during parsing %s', value.recipe,
2357 exc_info=(etype, value, tb))
Andrew Geissler9aee5002022-03-30 16:27:02 +00002358 self.shutdown(clean=False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002359 return False
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002360 except Exception as exc:
2361 self.error += 1
2362 etype, value, tb = sys.exc_info()
2363 if hasattr(value, "recipe"):
Patrick Williamsc0f7c042017-02-23 20:41:17 -06002364 logger.error('Unable to parse %s' % value.recipe,
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002365 exc_info=(etype, value, exc.traceback))
2366 else:
2367 # Most likely, an exception occurred during raising an exception
2368 import traceback
2369 logger.error('Exception during parse: %s' % traceback.format_exc())
Andrew Geissler9aee5002022-03-30 16:27:02 +00002370 self.shutdown(clean=False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002371 return False
2372
2373 self.current += 1
2374 self.virtuals += len(result)
2375 if parsed:
2376 self.parsed += 1
2377 if self.parsed % self.progress_chunk == 0:
2378 bb.event.fire(bb.event.ParseProgress(self.parsed, self.toparse),
2379 self.cfgdata)
2380 else:
2381 self.cached += 1
2382
2383 for virtualfn, info_array in result:
2384 if info_array[0].skipped:
2385 self.skipped += 1
2386 self.cooker.skiplist[virtualfn] = SkippedPackage(info_array[0])
Andrew Geissler5a43b432020-06-13 10:46:56 -05002387 self.bb_caches[mc].add_info(virtualfn, info_array, self.cooker.recipecaches[mc],
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002388 parsed=parsed, watcher = self.cooker.add_filewatch)
2389 return True
2390
2391 def reparse(self, filename):
Andrew Geissler517393d2023-01-13 08:55:19 -06002392 bb.cache.SiggenRecipeInfo.reset()
Andrew Geissler5a43b432020-06-13 10:46:56 -05002393 to_reparse = set()
2394 for mc in self.cooker.multiconfigs:
2395 to_reparse.add((mc, filename, self.cooker.collections[mc].get_file_appends(filename)))
2396
2397 for mc, filename, appends in to_reparse:
2398 infos = self.bb_caches[mc].parse(filename, appends)
2399 for vfn, info_array in infos:
2400 self.cooker.recipecaches[mc].add_from_recipeinfo(vfn, info_array)