blob: 516d45e4aabdc8465c866d8c4d6707b51c23ba5b [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001#
2# Copyright (C) 2003, 2004 Chris Larson
3# Copyright (C) 2003, 2004 Phil Blundell
4# Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
5# Copyright (C) 2005 Holger Hans Peter Freyther
6# Copyright (C) 2005 ROAD GmbH
7# Copyright (C) 2006 Richard Purdie
8#
Brad Bishopc342db32019-05-15 21:57:59 -04009# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -050010#
Patrick Williamsc124f4f2015-09-15 14:41:29 -050011
12import re
13import logging
14from bb import data, utils
15from collections import defaultdict
16import bb
17
18logger = logging.getLogger("BitBake.Provider")
19
20class NoProvider(bb.BBHandledException):
21 """Exception raised when no provider of a build dependency can be found"""
22
23class NoRProvider(bb.BBHandledException):
24 """Exception raised when no provider of a runtime dependency can be found"""
25
26class MultipleRProvider(bb.BBHandledException):
27 """Exception raised when multiple providers of a runtime dependency can be found"""
28
29def findProviders(cfgData, dataCache, pkg_pn = None):
30 """
31 Convenience function to get latest and preferred providers in pkg_pn
32 """
33
34 if not pkg_pn:
35 pkg_pn = dataCache.pkg_pn
36
37 # Need to ensure data store is expanded
38 localdata = data.createCopy(cfgData)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050039 bb.data.expandKeys(localdata)
40
Andrew Geissler95ac1b82021-03-31 14:34:31 -050041 required = {}
Patrick Williamsc124f4f2015-09-15 14:41:29 -050042 preferred_versions = {}
43 latest_versions = {}
44
45 for pn in pkg_pn:
Andrew Geissler95ac1b82021-03-31 14:34:31 -050046 (last_ver, last_file, pref_ver, pref_file, req) = findBestProvider(pn, localdata, dataCache, pkg_pn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050047 preferred_versions[pn] = (pref_ver, pref_file)
48 latest_versions[pn] = (last_ver, last_file)
Andrew Geissler95ac1b82021-03-31 14:34:31 -050049 required[pn] = req
Patrick Williamsc124f4f2015-09-15 14:41:29 -050050
Andrew Geissler95ac1b82021-03-31 14:34:31 -050051 return (latest_versions, preferred_versions, required)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050052
53def allProviders(dataCache):
54 """
55 Find all providers for each pn
56 """
57 all_providers = defaultdict(list)
58 for (fn, pn) in dataCache.pkg_fn.items():
59 ver = dataCache.pkg_pepvpr[fn]
60 all_providers[pn].append((ver, fn))
61 return all_providers
62
Patrick Williamsc124f4f2015-09-15 14:41:29 -050063def sortPriorities(pn, dataCache, pkg_pn = None):
64 """
65 Reorder pkg_pn by file priority and default preference
66 """
67
68 if not pkg_pn:
69 pkg_pn = dataCache.pkg_pn
70
71 files = pkg_pn[pn]
72 priorities = {}
73 for f in files:
74 priority = dataCache.bbfile_priority[f]
75 preference = dataCache.pkg_dp[f]
76 if priority not in priorities:
77 priorities[priority] = {}
78 if preference not in priorities[priority]:
79 priorities[priority][preference] = []
80 priorities[priority][preference].append(f)
81 tmp_pn = []
82 for pri in sorted(priorities):
83 tmp_pref = []
84 for pref in sorted(priorities[pri]):
85 tmp_pref.extend(priorities[pri][pref])
86 tmp_pn = [tmp_pref] + tmp_pn
87
88 return tmp_pn
89
Andrew Geissler95ac1b82021-03-31 14:34:31 -050090def versionVariableMatch(cfgData, keyword, pn):
91 """
92 Return the value of the <keyword>_VERSION variable if set.
93 """
94
95 # pn can contain '_', e.g. gcc-cross-x86_64 and an override cannot
96 # hence we do this manually rather than use OVERRIDES
97 ver = cfgData.getVar("%s_VERSION_pn-%s" % (keyword, pn))
98 if not ver:
99 ver = cfgData.getVar("%s_VERSION_%s" % (keyword, pn))
100 if not ver:
101 ver = cfgData.getVar("%s_VERSION" % keyword)
102
103 return ver
104
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500105def preferredVersionMatch(pe, pv, pr, preferred_e, preferred_v, preferred_r):
106 """
107 Check if the version pe,pv,pr is the preferred one.
108 If there is preferred version defined and ends with '%', then pv has to start with that version after removing the '%'
109 """
Andrew Geissler82c905d2020-04-13 13:39:40 -0500110 if pr == preferred_r or preferred_r is None:
111 if pe == preferred_e or preferred_e is None:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500112 if preferred_v == pv:
113 return True
Andrew Geissler82c905d2020-04-13 13:39:40 -0500114 if preferred_v is not None and preferred_v.endswith('%') and pv.startswith(preferred_v[:len(preferred_v)-1]):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500115 return True
116 return False
117
118def findPreferredProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
119 """
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500120 Find the first provider in pkg_pn with REQUIRED_VERSION or PREFERRED_VERSION set.
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500121 """
122
123 preferred_file = None
124 preferred_ver = None
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500125 required = False
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500126
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500127 required_v = versionVariableMatch(cfgData, "REQUIRED", pn)
128 preferred_v = versionVariableMatch(cfgData, "PREFERRED", pn)
129
130 itemstr = ""
131 if item:
132 itemstr = " (for item %s)" % item
133
134 if required_v is not None:
135 if preferred_v is not None:
Andrew Geissler5f350902021-07-23 13:09:54 -0400136 logger.warning("REQUIRED_VERSION and PREFERRED_VERSION for package %s%s are both set using REQUIRED_VERSION %s", pn, itemstr, required_v)
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500137 else:
138 logger.debug("REQUIRED_VERSION is set for package %s%s", pn, itemstr)
139 # REQUIRED_VERSION always takes precedence over PREFERRED_VERSION
140 preferred_v = required_v
141 required = True
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500142
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500143 if preferred_v:
Brad Bishop19323692019-04-05 15:28:33 -0400144 m = re.match(r'(\d+:)*(.*)(_.*)*', preferred_v)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500145 if m:
146 if m.group(1):
147 preferred_e = m.group(1)[:-1]
148 else:
149 preferred_e = None
150 preferred_v = m.group(2)
151 if m.group(3):
152 preferred_r = m.group(3)[1:]
153 else:
154 preferred_r = None
155 else:
156 preferred_e = None
157 preferred_r = None
158
159 for file_set in pkg_pn:
160 for f in file_set:
161 pe, pv, pr = dataCache.pkg_pepvpr[f]
162 if preferredVersionMatch(pe, pv, pr, preferred_e, preferred_v, preferred_r):
163 preferred_file = f
164 preferred_ver = (pe, pv, pr)
165 break
166 if preferred_file:
167 break;
168 if preferred_r:
169 pv_str = '%s-%s' % (preferred_v, preferred_r)
170 else:
171 pv_str = preferred_v
172 if not (preferred_e is None):
173 pv_str = '%s:%s' % (preferred_e, pv_str)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500174 if preferred_file is None:
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500175 if not required:
Andrew Geissler5f350902021-07-23 13:09:54 -0400176 logger.warning("preferred version %s of %s not available%s", pv_str, pn, itemstr)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500177 available_vers = []
178 for file_set in pkg_pn:
179 for f in file_set:
180 pe, pv, pr = dataCache.pkg_pepvpr[f]
181 ver_str = pv
182 if pe:
183 ver_str = "%s:%s" % (pe, ver_str)
184 if not ver_str in available_vers:
185 available_vers.append(ver_str)
186 if available_vers:
187 available_vers.sort()
Andrew Geissler5f350902021-07-23 13:09:54 -0400188 logger.warning("versions of %s available: %s", pn, ' '.join(available_vers))
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500189 if required:
190 logger.error("required version %s of %s not available%s", pv_str, pn, itemstr)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500191 else:
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500192 if required:
193 logger.debug("selecting %s as REQUIRED_VERSION %s of package %s%s", preferred_file, pv_str, pn, itemstr)
194 else:
195 logger.debug("selecting %s as PREFERRED_VERSION %s of package %s%s", preferred_file, pv_str, pn, itemstr)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500196
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500197 return (preferred_ver, preferred_file, required)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500198
199def findLatestProvider(pn, cfgData, dataCache, file_set):
200 """
201 Return the highest version of the providers in file_set.
202 Take default preferences into account.
203 """
204 latest = None
205 latest_p = 0
206 latest_f = None
207 for file_name in file_set:
208 pe, pv, pr = dataCache.pkg_pepvpr[file_name]
209 dp = dataCache.pkg_dp[file_name]
210
211 if (latest is None) or ((latest_p == dp) and (utils.vercmp(latest, (pe, pv, pr)) < 0)) or (dp > latest_p):
212 latest = (pe, pv, pr)
213 latest_f = file_name
214 latest_p = dp
215
216 return (latest, latest_f)
217
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500218def findBestProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
219 """
220 If there is a PREFERRED_VERSION, find the highest-priority bbfile
221 providing that version. If not, find the latest version provided by
222 an bbfile in the highest-priority set.
223 """
224
225 sortpkg_pn = sortPriorities(pn, dataCache, pkg_pn)
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500226 # Find the highest priority provider with a REQUIRED_VERSION or PREFERRED_VERSION set
227 (preferred_ver, preferred_file, required) = findPreferredProvider(pn, cfgData, dataCache, sortpkg_pn, item)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500228 # Find the latest version of the highest priority provider
229 (latest, latest_f) = findLatestProvider(pn, cfgData, dataCache, sortpkg_pn[0])
230
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500231 if not required and preferred_file is None:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500232 preferred_file = latest_f
233 preferred_ver = latest
234
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500235 return (latest, latest_f, preferred_ver, preferred_file, required)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500236
237def _filterProviders(providers, item, cfgData, dataCache):
238 """
239 Take a list of providers and filter/reorder according to the
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500240 environment variables
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500241 """
242 eligible = []
243 preferred_versions = {}
244 sortpkg_pn = {}
245
246 # The order of providers depends on the order of the files on the disk
247 # up to here. Sort pkg_pn to make dependency issues reproducible rather
248 # than effectively random.
249 providers.sort()
250
251 # Collate providers by PN
252 pkg_pn = {}
253 for p in providers:
254 pn = dataCache.pkg_fn[p]
255 if pn not in pkg_pn:
256 pkg_pn[pn] = []
257 pkg_pn[pn].append(p)
258
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600259 logger.debug("providers for %s are: %s", item, list(sorted(pkg_pn.keys())))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500260
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500261 # First add REQUIRED_VERSIONS or PREFERRED_VERSIONS
Brad Bishop00111322018-04-01 22:23:53 -0400262 for pn in sorted(pkg_pn):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500263 sortpkg_pn[pn] = sortPriorities(pn, dataCache, pkg_pn)
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500264 preferred_ver, preferred_file, required = findPreferredProvider(pn, cfgData, dataCache, sortpkg_pn[pn], item)
265 if required and preferred_file is None:
266 return eligible
267 preferred_versions[pn] = (preferred_ver, preferred_file)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500268 if preferred_versions[pn][1]:
269 eligible.append(preferred_versions[pn][1])
270
271 # Now add latest versions
Brad Bishop00111322018-04-01 22:23:53 -0400272 for pn in sorted(sortpkg_pn):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500273 if pn in preferred_versions and preferred_versions[pn][1]:
274 continue
275 preferred_versions[pn] = findLatestProvider(pn, cfgData, dataCache, sortpkg_pn[pn][0])
276 eligible.append(preferred_versions[pn][1])
277
Andrew Geissler90fd73c2021-03-05 15:25:55 -0600278 if not eligible:
Andrew Geissler90fd73c2021-03-05 15:25:55 -0600279 return eligible
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500280
281 # If pn == item, give it a slight default preference
282 # This means PREFERRED_PROVIDER_foobar defaults to foobar if available
283 for p in providers:
284 pn = dataCache.pkg_fn[p]
285 if pn != item:
286 continue
287 (newvers, fn) = preferred_versions[pn]
288 if not fn in eligible:
289 continue
290 eligible.remove(fn)
291 eligible = [fn] + eligible
292
293 return eligible
294
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500295def filterProviders(providers, item, cfgData, dataCache):
296 """
297 Take a list of providers and filter/reorder according to the
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500298 environment variables
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500299 Takes a "normal" target item
300 """
301
302 eligible = _filterProviders(providers, item, cfgData, dataCache)
303
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500304 prefervar = cfgData.getVar('PREFERRED_PROVIDER_%s' % item)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500305 if prefervar:
306 dataCache.preferred[item] = prefervar
307
308 foundUnique = False
309 if item in dataCache.preferred:
310 for p in eligible:
311 pn = dataCache.pkg_fn[p]
312 if dataCache.preferred[item] == pn:
313 logger.verbose("selecting %s to satisfy %s due to PREFERRED_PROVIDERS", pn, item)
314 eligible.remove(p)
315 eligible = [p] + eligible
316 foundUnique = True
317 break
318
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600319 logger.debug("sorted providers for %s are: %s", item, eligible)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500320
321 return eligible, foundUnique
322
323def filterProvidersRunTime(providers, item, cfgData, dataCache):
324 """
325 Take a list of providers and filter/reorder according to the
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500326 environment variables
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500327 Takes a "runtime" target item
328 """
329
330 eligible = _filterProviders(providers, item, cfgData, dataCache)
331
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500332 # First try and match any PREFERRED_RPROVIDER entry
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500333 prefervar = cfgData.getVar('PREFERRED_RPROVIDER_%s' % item)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500334 foundUnique = False
335 if prefervar:
336 for p in eligible:
337 pn = dataCache.pkg_fn[p]
338 if prefervar == pn:
339 logger.verbose("selecting %s to satisfy %s due to PREFERRED_RPROVIDER", pn, item)
340 eligible.remove(p)
341 eligible = [p] + eligible
342 foundUnique = True
343 numberPreferred = 1
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500344 break
345
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500346 # If we didn't find an RPROVIDER entry, try and infer the provider from PREFERRED_PROVIDER entries
347 # by looking through the provides of each eligible recipe and seeing if a PREFERRED_PROVIDER was set.
348 # This is most useful for virtual/ entries rather than having a RPROVIDER per entry.
349 if not foundUnique:
350 # Should use dataCache.preferred here?
351 preferred = []
352 preferred_vars = []
353 pns = {}
354 for p in eligible:
355 pns[dataCache.pkg_fn[p]] = p
356 for p in eligible:
357 pn = dataCache.pkg_fn[p]
358 provides = dataCache.pn_provides[pn]
359 for provide in provides:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500360 prefervar = cfgData.getVar('PREFERRED_PROVIDER_%s' % provide)
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600361 #logger.debug("checking PREFERRED_PROVIDER_%s (value %s) against %s", provide, prefervar, pns.keys())
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500362 if prefervar in pns and pns[prefervar] not in preferred:
363 var = "PREFERRED_PROVIDER_%s = %s" % (provide, prefervar)
364 logger.verbose("selecting %s to satisfy runtime %s due to %s", prefervar, item, var)
365 preferred_vars.append(var)
366 pref = pns[prefervar]
367 eligible.remove(pref)
368 eligible = [pref] + eligible
369 preferred.append(pref)
370 break
371
372 numberPreferred = len(preferred)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500373
374 if numberPreferred > 1:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500375 logger.error("Trying to resolve runtime dependency %s resulted in conflicting PREFERRED_PROVIDER entries being found.\nThe providers found were: %s\nThe PREFERRED_PROVIDER entries resulting in this conflict were: %s. You could set PREFERRED_RPROVIDER_%s" % (item, preferred, preferred_vars, item))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500376
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600377 logger.debug("sorted runtime providers for %s are: %s", item, eligible)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500378
379 return eligible, numberPreferred
380
381regexp_cache = {}
382
383def getRuntimeProviders(dataCache, rdepend):
384 """
385 Return any providers of runtime dependency
386 """
387 rproviders = []
388
389 if rdepend in dataCache.rproviders:
390 rproviders += dataCache.rproviders[rdepend]
391
392 if rdepend in dataCache.packages:
393 rproviders += dataCache.packages[rdepend]
394
395 if rproviders:
396 return rproviders
397
398 # Only search dynamic packages if we can't find anything in other variables
399 for pattern in dataCache.packages_dynamic:
Brad Bishop19323692019-04-05 15:28:33 -0400400 pattern = pattern.replace(r'+', r"\+")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500401 if pattern in regexp_cache:
402 regexp = regexp_cache[pattern]
403 else:
404 try:
405 regexp = re.compile(pattern)
406 except:
407 logger.error("Error parsing regular expression '%s'", pattern)
408 raise
409 regexp_cache[pattern] = regexp
410 if regexp.match(rdepend):
411 rproviders += dataCache.packages_dynamic[pattern]
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600412 logger.debug("Assuming %s is a dynamic package, but it may not exist" % rdepend)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500413
414 return rproviders
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500415
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600416def buildWorldTargetList(dataCache, task=None):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500417 """
418 Build package list for "bitbake world"
419 """
420 if dataCache.world_target:
421 return
422
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600423 logger.debug("collating packages for \"world\"")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500424 for f in dataCache.possible_world:
425 terminal = True
426 pn = dataCache.pkg_fn[f]
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600427 if task and task not in dataCache.task_deps[f]['tasks']:
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600428 logger.debug2("World build skipping %s as task %s doesn't exist", f, task)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600429 terminal = False
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500430
431 for p in dataCache.pn_provides[pn]:
432 if p.startswith('virtual/'):
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600433 logger.debug2("World build skipping %s due to %s provider starting with virtual/", f, p)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500434 terminal = False
435 break
436 for pf in dataCache.providers[p]:
437 if dataCache.pkg_fn[pf] != pn:
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600438 logger.debug2("World build skipping %s due to both us and %s providing %s", f, pf, p)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500439 terminal = False
440 break
441 if terminal:
442 dataCache.world_target.add(pn)