blob: 3f66a3d99fa9d4dd1c97a4e44bc6f172fb7860ef [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001#
2# Copyright (C) 2003, 2004 Chris Larson
3# Copyright (C) 2003, 2004 Phil Blundell
4# Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
5# Copyright (C) 2005 Holger Hans Peter Freyther
6# Copyright (C) 2005 ROAD GmbH
7# Copyright (C) 2006 Richard Purdie
8#
Brad Bishopc342db32019-05-15 21:57:59 -04009# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -050010#
Patrick Williamsc124f4f2015-09-15 14:41:29 -050011
12import re
13import logging
14from bb import data, utils
15from collections import defaultdict
16import bb
17
18logger = logging.getLogger("BitBake.Provider")
19
20class NoProvider(bb.BBHandledException):
21 """Exception raised when no provider of a build dependency can be found"""
22
23class NoRProvider(bb.BBHandledException):
24 """Exception raised when no provider of a runtime dependency can be found"""
25
26class MultipleRProvider(bb.BBHandledException):
27 """Exception raised when multiple providers of a runtime dependency can be found"""
28
29def findProviders(cfgData, dataCache, pkg_pn = None):
30 """
31 Convenience function to get latest and preferred providers in pkg_pn
32 """
33
34 if not pkg_pn:
35 pkg_pn = dataCache.pkg_pn
36
37 # Need to ensure data store is expanded
38 localdata = data.createCopy(cfgData)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050039 bb.data.expandKeys(localdata)
40
41 preferred_versions = {}
42 latest_versions = {}
43
44 for pn in pkg_pn:
45 (last_ver, last_file, pref_ver, pref_file) = findBestProvider(pn, localdata, dataCache, pkg_pn)
46 preferred_versions[pn] = (pref_ver, pref_file)
47 latest_versions[pn] = (last_ver, last_file)
48
49 return (latest_versions, preferred_versions)
50
51
52def allProviders(dataCache):
53 """
54 Find all providers for each pn
55 """
56 all_providers = defaultdict(list)
57 for (fn, pn) in dataCache.pkg_fn.items():
58 ver = dataCache.pkg_pepvpr[fn]
59 all_providers[pn].append((ver, fn))
60 return all_providers
61
62
63def sortPriorities(pn, dataCache, pkg_pn = None):
64 """
65 Reorder pkg_pn by file priority and default preference
66 """
67
68 if not pkg_pn:
69 pkg_pn = dataCache.pkg_pn
70
71 files = pkg_pn[pn]
72 priorities = {}
73 for f in files:
74 priority = dataCache.bbfile_priority[f]
75 preference = dataCache.pkg_dp[f]
76 if priority not in priorities:
77 priorities[priority] = {}
78 if preference not in priorities[priority]:
79 priorities[priority][preference] = []
80 priorities[priority][preference].append(f)
81 tmp_pn = []
82 for pri in sorted(priorities):
83 tmp_pref = []
84 for pref in sorted(priorities[pri]):
85 tmp_pref.extend(priorities[pri][pref])
86 tmp_pn = [tmp_pref] + tmp_pn
87
88 return tmp_pn
89
90def preferredVersionMatch(pe, pv, pr, preferred_e, preferred_v, preferred_r):
91 """
92 Check if the version pe,pv,pr is the preferred one.
93 If there is preferred version defined and ends with '%', then pv has to start with that version after removing the '%'
94 """
Andrew Geissler82c905d2020-04-13 13:39:40 -050095 if pr == preferred_r or preferred_r is None:
96 if pe == preferred_e or preferred_e is None:
Patrick Williamsc124f4f2015-09-15 14:41:29 -050097 if preferred_v == pv:
98 return True
Andrew Geissler82c905d2020-04-13 13:39:40 -050099 if preferred_v is not None and preferred_v.endswith('%') and pv.startswith(preferred_v[:len(preferred_v)-1]):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500100 return True
101 return False
102
103def findPreferredProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
104 """
105 Find the first provider in pkg_pn with a PREFERRED_VERSION set.
106 """
107
108 preferred_file = None
109 preferred_ver = None
110
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500111 # pn can contain '_', e.g. gcc-cross-x86_64 and an override cannot
112 # hence we do this manually rather than use OVERRIDES
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500113 preferred_v = cfgData.getVar("PREFERRED_VERSION_pn-%s" % pn)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500114 if not preferred_v:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500115 preferred_v = cfgData.getVar("PREFERRED_VERSION_%s" % pn)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500116 if not preferred_v:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500117 preferred_v = cfgData.getVar("PREFERRED_VERSION")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500118
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500119 if preferred_v:
Brad Bishop19323692019-04-05 15:28:33 -0400120 m = re.match(r'(\d+:)*(.*)(_.*)*', preferred_v)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500121 if m:
122 if m.group(1):
123 preferred_e = m.group(1)[:-1]
124 else:
125 preferred_e = None
126 preferred_v = m.group(2)
127 if m.group(3):
128 preferred_r = m.group(3)[1:]
129 else:
130 preferred_r = None
131 else:
132 preferred_e = None
133 preferred_r = None
134
135 for file_set in pkg_pn:
136 for f in file_set:
137 pe, pv, pr = dataCache.pkg_pepvpr[f]
138 if preferredVersionMatch(pe, pv, pr, preferred_e, preferred_v, preferred_r):
139 preferred_file = f
140 preferred_ver = (pe, pv, pr)
141 break
142 if preferred_file:
143 break;
144 if preferred_r:
145 pv_str = '%s-%s' % (preferred_v, preferred_r)
146 else:
147 pv_str = preferred_v
148 if not (preferred_e is None):
149 pv_str = '%s:%s' % (preferred_e, pv_str)
150 itemstr = ""
151 if item:
152 itemstr = " (for item %s)" % item
153 if preferred_file is None:
Andrew Geissler6ce62a22020-11-30 19:58:47 -0600154 logger.warn("preferred version %s of %s not available%s", pv_str, pn, itemstr)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500155 available_vers = []
156 for file_set in pkg_pn:
157 for f in file_set:
158 pe, pv, pr = dataCache.pkg_pepvpr[f]
159 ver_str = pv
160 if pe:
161 ver_str = "%s:%s" % (pe, ver_str)
162 if not ver_str in available_vers:
163 available_vers.append(ver_str)
164 if available_vers:
165 available_vers.sort()
Andrew Geissler6ce62a22020-11-30 19:58:47 -0600166 logger.warn("versions of %s available: %s", pn, ' '.join(available_vers))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500167 else:
168 logger.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s%s", preferred_file, pv_str, pn, itemstr)
169
170 return (preferred_ver, preferred_file)
171
172
173def findLatestProvider(pn, cfgData, dataCache, file_set):
174 """
175 Return the highest version of the providers in file_set.
176 Take default preferences into account.
177 """
178 latest = None
179 latest_p = 0
180 latest_f = None
181 for file_name in file_set:
182 pe, pv, pr = dataCache.pkg_pepvpr[file_name]
183 dp = dataCache.pkg_dp[file_name]
184
185 if (latest is None) or ((latest_p == dp) and (utils.vercmp(latest, (pe, pv, pr)) < 0)) or (dp > latest_p):
186 latest = (pe, pv, pr)
187 latest_f = file_name
188 latest_p = dp
189
190 return (latest, latest_f)
191
192
193def findBestProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
194 """
195 If there is a PREFERRED_VERSION, find the highest-priority bbfile
196 providing that version. If not, find the latest version provided by
197 an bbfile in the highest-priority set.
198 """
199
200 sortpkg_pn = sortPriorities(pn, dataCache, pkg_pn)
201 # Find the highest priority provider with a PREFERRED_VERSION set
202 (preferred_ver, preferred_file) = findPreferredProvider(pn, cfgData, dataCache, sortpkg_pn, item)
203 # Find the latest version of the highest priority provider
204 (latest, latest_f) = findLatestProvider(pn, cfgData, dataCache, sortpkg_pn[0])
205
206 if preferred_file is None:
207 preferred_file = latest_f
208 preferred_ver = latest
209
210 return (latest, latest_f, preferred_ver, preferred_file)
211
212
213def _filterProviders(providers, item, cfgData, dataCache):
214 """
215 Take a list of providers and filter/reorder according to the
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500216 environment variables
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500217 """
218 eligible = []
219 preferred_versions = {}
220 sortpkg_pn = {}
221
222 # The order of providers depends on the order of the files on the disk
223 # up to here. Sort pkg_pn to make dependency issues reproducible rather
224 # than effectively random.
225 providers.sort()
226
227 # Collate providers by PN
228 pkg_pn = {}
229 for p in providers:
230 pn = dataCache.pkg_fn[p]
231 if pn not in pkg_pn:
232 pkg_pn[pn] = []
233 pkg_pn[pn].append(p)
234
Brad Bishop00111322018-04-01 22:23:53 -0400235 logger.debug(1, "providers for %s are: %s", item, list(sorted(pkg_pn.keys())))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500236
237 # First add PREFERRED_VERSIONS
Brad Bishop00111322018-04-01 22:23:53 -0400238 for pn in sorted(pkg_pn):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500239 sortpkg_pn[pn] = sortPriorities(pn, dataCache, pkg_pn)
240 preferred_versions[pn] = findPreferredProvider(pn, cfgData, dataCache, sortpkg_pn[pn], item)
241 if preferred_versions[pn][1]:
242 eligible.append(preferred_versions[pn][1])
243
244 # Now add latest versions
Brad Bishop00111322018-04-01 22:23:53 -0400245 for pn in sorted(sortpkg_pn):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500246 if pn in preferred_versions and preferred_versions[pn][1]:
247 continue
248 preferred_versions[pn] = findLatestProvider(pn, cfgData, dataCache, sortpkg_pn[pn][0])
249 eligible.append(preferred_versions[pn][1])
250
251 if len(eligible) == 0:
252 logger.error("no eligible providers for %s", item)
253 return 0
254
255 # If pn == item, give it a slight default preference
256 # This means PREFERRED_PROVIDER_foobar defaults to foobar if available
257 for p in providers:
258 pn = dataCache.pkg_fn[p]
259 if pn != item:
260 continue
261 (newvers, fn) = preferred_versions[pn]
262 if not fn in eligible:
263 continue
264 eligible.remove(fn)
265 eligible = [fn] + eligible
266
267 return eligible
268
269
270def filterProviders(providers, item, cfgData, dataCache):
271 """
272 Take a list of providers and filter/reorder according to the
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500273 environment variables
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500274 Takes a "normal" target item
275 """
276
277 eligible = _filterProviders(providers, item, cfgData, dataCache)
278
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500279 prefervar = cfgData.getVar('PREFERRED_PROVIDER_%s' % item)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500280 if prefervar:
281 dataCache.preferred[item] = prefervar
282
283 foundUnique = False
284 if item in dataCache.preferred:
285 for p in eligible:
286 pn = dataCache.pkg_fn[p]
287 if dataCache.preferred[item] == pn:
288 logger.verbose("selecting %s to satisfy %s due to PREFERRED_PROVIDERS", pn, item)
289 eligible.remove(p)
290 eligible = [p] + eligible
291 foundUnique = True
292 break
293
294 logger.debug(1, "sorted providers for %s are: %s", item, eligible)
295
296 return eligible, foundUnique
297
298def filterProvidersRunTime(providers, item, cfgData, dataCache):
299 """
300 Take a list of providers and filter/reorder according to the
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500301 environment variables
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500302 Takes a "runtime" target item
303 """
304
305 eligible = _filterProviders(providers, item, cfgData, dataCache)
306
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500307 # First try and match any PREFERRED_RPROVIDER entry
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500308 prefervar = cfgData.getVar('PREFERRED_RPROVIDER_%s' % item)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500309 foundUnique = False
310 if prefervar:
311 for p in eligible:
312 pn = dataCache.pkg_fn[p]
313 if prefervar == pn:
314 logger.verbose("selecting %s to satisfy %s due to PREFERRED_RPROVIDER", pn, item)
315 eligible.remove(p)
316 eligible = [p] + eligible
317 foundUnique = True
318 numberPreferred = 1
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500319 break
320
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500321 # If we didn't find an RPROVIDER entry, try and infer the provider from PREFERRED_PROVIDER entries
322 # by looking through the provides of each eligible recipe and seeing if a PREFERRED_PROVIDER was set.
323 # This is most useful for virtual/ entries rather than having a RPROVIDER per entry.
324 if not foundUnique:
325 # Should use dataCache.preferred here?
326 preferred = []
327 preferred_vars = []
328 pns = {}
329 for p in eligible:
330 pns[dataCache.pkg_fn[p]] = p
331 for p in eligible:
332 pn = dataCache.pkg_fn[p]
333 provides = dataCache.pn_provides[pn]
334 for provide in provides:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500335 prefervar = cfgData.getVar('PREFERRED_PROVIDER_%s' % provide)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500336 #logger.debug(1, "checking PREFERRED_PROVIDER_%s (value %s) against %s", provide, prefervar, pns.keys())
337 if prefervar in pns and pns[prefervar] not in preferred:
338 var = "PREFERRED_PROVIDER_%s = %s" % (provide, prefervar)
339 logger.verbose("selecting %s to satisfy runtime %s due to %s", prefervar, item, var)
340 preferred_vars.append(var)
341 pref = pns[prefervar]
342 eligible.remove(pref)
343 eligible = [pref] + eligible
344 preferred.append(pref)
345 break
346
347 numberPreferred = len(preferred)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500348
349 if numberPreferred > 1:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500350 logger.error("Trying to resolve runtime dependency %s resulted in conflicting PREFERRED_PROVIDER entries being found.\nThe providers found were: %s\nThe PREFERRED_PROVIDER entries resulting in this conflict were: %s. You could set PREFERRED_RPROVIDER_%s" % (item, preferred, preferred_vars, item))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500351
352 logger.debug(1, "sorted runtime providers for %s are: %s", item, eligible)
353
354 return eligible, numberPreferred
355
356regexp_cache = {}
357
358def getRuntimeProviders(dataCache, rdepend):
359 """
360 Return any providers of runtime dependency
361 """
362 rproviders = []
363
364 if rdepend in dataCache.rproviders:
365 rproviders += dataCache.rproviders[rdepend]
366
367 if rdepend in dataCache.packages:
368 rproviders += dataCache.packages[rdepend]
369
370 if rproviders:
371 return rproviders
372
373 # Only search dynamic packages if we can't find anything in other variables
374 for pattern in dataCache.packages_dynamic:
Brad Bishop19323692019-04-05 15:28:33 -0400375 pattern = pattern.replace(r'+', r"\+")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500376 if pattern in regexp_cache:
377 regexp = regexp_cache[pattern]
378 else:
379 try:
380 regexp = re.compile(pattern)
381 except:
382 logger.error("Error parsing regular expression '%s'", pattern)
383 raise
384 regexp_cache[pattern] = regexp
385 if regexp.match(rdepend):
386 rproviders += dataCache.packages_dynamic[pattern]
387 logger.debug(1, "Assuming %s is a dynamic package, but it may not exist" % rdepend)
388
389 return rproviders
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500390
391
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600392def buildWorldTargetList(dataCache, task=None):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500393 """
394 Build package list for "bitbake world"
395 """
396 if dataCache.world_target:
397 return
398
399 logger.debug(1, "collating packages for \"world\"")
400 for f in dataCache.possible_world:
401 terminal = True
402 pn = dataCache.pkg_fn[f]
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600403 if task and task not in dataCache.task_deps[f]['tasks']:
404 logger.debug(2, "World build skipping %s as task %s doesn't exist", f, task)
405 terminal = False
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500406
407 for p in dataCache.pn_provides[pn]:
408 if p.startswith('virtual/'):
409 logger.debug(2, "World build skipping %s due to %s provider starting with virtual/", f, p)
410 terminal = False
411 break
412 for pf in dataCache.providers[p]:
413 if dataCache.pkg_fn[pf] != pn:
414 logger.debug(2, "World build skipping %s due to both us and %s providing %s", f, pf, p)
415 terminal = False
416 break
417 if terminal:
418 dataCache.world_target.add(pn)