blob: 443187e17fb891189b34aebb4f229709c7e4987b [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# ex:ts=4:sw=4:sts=4:et
2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
3#
4# Copyright (C) 2003, 2004 Chris Larson
5# Copyright (C) 2003, 2004 Phil Blundell
6# Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
7# Copyright (C) 2005 Holger Hans Peter Freyther
8# Copyright (C) 2005 ROAD GmbH
9# Copyright (C) 2006 Richard Purdie
10#
11# This program is free software; you can redistribute it and/or modify
12# it under the terms of the GNU General Public License version 2 as
13# published by the Free Software Foundation.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License along
21# with this program; if not, write to the Free Software Foundation, Inc.,
22# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
23
24import re
25import logging
26from bb import data, utils
27from collections import defaultdict
28import bb
29
30logger = logging.getLogger("BitBake.Provider")
31
32class NoProvider(bb.BBHandledException):
33 """Exception raised when no provider of a build dependency can be found"""
34
35class NoRProvider(bb.BBHandledException):
36 """Exception raised when no provider of a runtime dependency can be found"""
37
38class MultipleRProvider(bb.BBHandledException):
39 """Exception raised when multiple providers of a runtime dependency can be found"""
40
41def findProviders(cfgData, dataCache, pkg_pn = None):
42 """
43 Convenience function to get latest and preferred providers in pkg_pn
44 """
45
46 if not pkg_pn:
47 pkg_pn = dataCache.pkg_pn
48
49 # Need to ensure data store is expanded
50 localdata = data.createCopy(cfgData)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050051 bb.data.expandKeys(localdata)
52
53 preferred_versions = {}
54 latest_versions = {}
55
56 for pn in pkg_pn:
57 (last_ver, last_file, pref_ver, pref_file) = findBestProvider(pn, localdata, dataCache, pkg_pn)
58 preferred_versions[pn] = (pref_ver, pref_file)
59 latest_versions[pn] = (last_ver, last_file)
60
61 return (latest_versions, preferred_versions)
62
63
64def allProviders(dataCache):
65 """
66 Find all providers for each pn
67 """
68 all_providers = defaultdict(list)
69 for (fn, pn) in dataCache.pkg_fn.items():
70 ver = dataCache.pkg_pepvpr[fn]
71 all_providers[pn].append((ver, fn))
72 return all_providers
73
74
75def sortPriorities(pn, dataCache, pkg_pn = None):
76 """
77 Reorder pkg_pn by file priority and default preference
78 """
79
80 if not pkg_pn:
81 pkg_pn = dataCache.pkg_pn
82
83 files = pkg_pn[pn]
84 priorities = {}
85 for f in files:
86 priority = dataCache.bbfile_priority[f]
87 preference = dataCache.pkg_dp[f]
88 if priority not in priorities:
89 priorities[priority] = {}
90 if preference not in priorities[priority]:
91 priorities[priority][preference] = []
92 priorities[priority][preference].append(f)
93 tmp_pn = []
94 for pri in sorted(priorities):
95 tmp_pref = []
96 for pref in sorted(priorities[pri]):
97 tmp_pref.extend(priorities[pri][pref])
98 tmp_pn = [tmp_pref] + tmp_pn
99
100 return tmp_pn
101
102def preferredVersionMatch(pe, pv, pr, preferred_e, preferred_v, preferred_r):
103 """
104 Check if the version pe,pv,pr is the preferred one.
105 If there is preferred version defined and ends with '%', then pv has to start with that version after removing the '%'
106 """
107 if (pr == preferred_r or preferred_r == None):
108 if (pe == preferred_e or preferred_e == None):
109 if preferred_v == pv:
110 return True
111 if preferred_v != None and preferred_v.endswith('%') and pv.startswith(preferred_v[:len(preferred_v)-1]):
112 return True
113 return False
114
115def findPreferredProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
116 """
117 Find the first provider in pkg_pn with a PREFERRED_VERSION set.
118 """
119
120 preferred_file = None
121 preferred_ver = None
122
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500123 # pn can contain '_', e.g. gcc-cross-x86_64 and an override cannot
124 # hence we do this manually rather than use OVERRIDES
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500125 preferred_v = cfgData.getVar("PREFERRED_VERSION_pn-%s" % pn)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500126 if not preferred_v:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500127 preferred_v = cfgData.getVar("PREFERRED_VERSION_%s" % pn)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500128 if not preferred_v:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500129 preferred_v = cfgData.getVar("PREFERRED_VERSION")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500130
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500131 if preferred_v:
132 m = re.match('(\d+:)*(.*)(_.*)*', preferred_v)
133 if m:
134 if m.group(1):
135 preferred_e = m.group(1)[:-1]
136 else:
137 preferred_e = None
138 preferred_v = m.group(2)
139 if m.group(3):
140 preferred_r = m.group(3)[1:]
141 else:
142 preferred_r = None
143 else:
144 preferred_e = None
145 preferred_r = None
146
147 for file_set in pkg_pn:
148 for f in file_set:
149 pe, pv, pr = dataCache.pkg_pepvpr[f]
150 if preferredVersionMatch(pe, pv, pr, preferred_e, preferred_v, preferred_r):
151 preferred_file = f
152 preferred_ver = (pe, pv, pr)
153 break
154 if preferred_file:
155 break;
156 if preferred_r:
157 pv_str = '%s-%s' % (preferred_v, preferred_r)
158 else:
159 pv_str = preferred_v
160 if not (preferred_e is None):
161 pv_str = '%s:%s' % (preferred_e, pv_str)
162 itemstr = ""
163 if item:
164 itemstr = " (for item %s)" % item
165 if preferred_file is None:
166 logger.info("preferred version %s of %s not available%s", pv_str, pn, itemstr)
167 available_vers = []
168 for file_set in pkg_pn:
169 for f in file_set:
170 pe, pv, pr = dataCache.pkg_pepvpr[f]
171 ver_str = pv
172 if pe:
173 ver_str = "%s:%s" % (pe, ver_str)
174 if not ver_str in available_vers:
175 available_vers.append(ver_str)
176 if available_vers:
177 available_vers.sort()
178 logger.info("versions of %s available: %s", pn, ' '.join(available_vers))
179 else:
180 logger.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s%s", preferred_file, pv_str, pn, itemstr)
181
182 return (preferred_ver, preferred_file)
183
184
185def findLatestProvider(pn, cfgData, dataCache, file_set):
186 """
187 Return the highest version of the providers in file_set.
188 Take default preferences into account.
189 """
190 latest = None
191 latest_p = 0
192 latest_f = None
193 for file_name in file_set:
194 pe, pv, pr = dataCache.pkg_pepvpr[file_name]
195 dp = dataCache.pkg_dp[file_name]
196
197 if (latest is None) or ((latest_p == dp) and (utils.vercmp(latest, (pe, pv, pr)) < 0)) or (dp > latest_p):
198 latest = (pe, pv, pr)
199 latest_f = file_name
200 latest_p = dp
201
202 return (latest, latest_f)
203
204
205def findBestProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
206 """
207 If there is a PREFERRED_VERSION, find the highest-priority bbfile
208 providing that version. If not, find the latest version provided by
209 an bbfile in the highest-priority set.
210 """
211
212 sortpkg_pn = sortPriorities(pn, dataCache, pkg_pn)
213 # Find the highest priority provider with a PREFERRED_VERSION set
214 (preferred_ver, preferred_file) = findPreferredProvider(pn, cfgData, dataCache, sortpkg_pn, item)
215 # Find the latest version of the highest priority provider
216 (latest, latest_f) = findLatestProvider(pn, cfgData, dataCache, sortpkg_pn[0])
217
218 if preferred_file is None:
219 preferred_file = latest_f
220 preferred_ver = latest
221
222 return (latest, latest_f, preferred_ver, preferred_file)
223
224
225def _filterProviders(providers, item, cfgData, dataCache):
226 """
227 Take a list of providers and filter/reorder according to the
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500228 environment variables
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500229 """
230 eligible = []
231 preferred_versions = {}
232 sortpkg_pn = {}
233
234 # The order of providers depends on the order of the files on the disk
235 # up to here. Sort pkg_pn to make dependency issues reproducible rather
236 # than effectively random.
237 providers.sort()
238
239 # Collate providers by PN
240 pkg_pn = {}
241 for p in providers:
242 pn = dataCache.pkg_fn[p]
243 if pn not in pkg_pn:
244 pkg_pn[pn] = []
245 pkg_pn[pn].append(p)
246
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600247 logger.debug(1, "providers for %s are: %s", item, list(pkg_pn.keys()))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500248
249 # First add PREFERRED_VERSIONS
250 for pn in pkg_pn:
251 sortpkg_pn[pn] = sortPriorities(pn, dataCache, pkg_pn)
252 preferred_versions[pn] = findPreferredProvider(pn, cfgData, dataCache, sortpkg_pn[pn], item)
253 if preferred_versions[pn][1]:
254 eligible.append(preferred_versions[pn][1])
255
256 # Now add latest versions
257 for pn in sortpkg_pn:
258 if pn in preferred_versions and preferred_versions[pn][1]:
259 continue
260 preferred_versions[pn] = findLatestProvider(pn, cfgData, dataCache, sortpkg_pn[pn][0])
261 eligible.append(preferred_versions[pn][1])
262
263 if len(eligible) == 0:
264 logger.error("no eligible providers for %s", item)
265 return 0
266
267 # If pn == item, give it a slight default preference
268 # This means PREFERRED_PROVIDER_foobar defaults to foobar if available
269 for p in providers:
270 pn = dataCache.pkg_fn[p]
271 if pn != item:
272 continue
273 (newvers, fn) = preferred_versions[pn]
274 if not fn in eligible:
275 continue
276 eligible.remove(fn)
277 eligible = [fn] + eligible
278
279 return eligible
280
281
282def filterProviders(providers, item, cfgData, dataCache):
283 """
284 Take a list of providers and filter/reorder according to the
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500285 environment variables
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500286 Takes a "normal" target item
287 """
288
289 eligible = _filterProviders(providers, item, cfgData, dataCache)
290
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500291 prefervar = cfgData.getVar('PREFERRED_PROVIDER_%s' % item)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500292 if prefervar:
293 dataCache.preferred[item] = prefervar
294
295 foundUnique = False
296 if item in dataCache.preferred:
297 for p in eligible:
298 pn = dataCache.pkg_fn[p]
299 if dataCache.preferred[item] == pn:
300 logger.verbose("selecting %s to satisfy %s due to PREFERRED_PROVIDERS", pn, item)
301 eligible.remove(p)
302 eligible = [p] + eligible
303 foundUnique = True
304 break
305
306 logger.debug(1, "sorted providers for %s are: %s", item, eligible)
307
308 return eligible, foundUnique
309
310def filterProvidersRunTime(providers, item, cfgData, dataCache):
311 """
312 Take a list of providers and filter/reorder according to the
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500313 environment variables
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500314 Takes a "runtime" target item
315 """
316
317 eligible = _filterProviders(providers, item, cfgData, dataCache)
318
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500319 # First try and match any PREFERRED_RPROVIDER entry
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500320 prefervar = cfgData.getVar('PREFERRED_RPROVIDER_%s' % item)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500321 foundUnique = False
322 if prefervar:
323 for p in eligible:
324 pn = dataCache.pkg_fn[p]
325 if prefervar == pn:
326 logger.verbose("selecting %s to satisfy %s due to PREFERRED_RPROVIDER", pn, item)
327 eligible.remove(p)
328 eligible = [p] + eligible
329 foundUnique = True
330 numberPreferred = 1
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500331 break
332
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500333 # If we didn't find an RPROVIDER entry, try and infer the provider from PREFERRED_PROVIDER entries
334 # by looking through the provides of each eligible recipe and seeing if a PREFERRED_PROVIDER was set.
335 # This is most useful for virtual/ entries rather than having a RPROVIDER per entry.
336 if not foundUnique:
337 # Should use dataCache.preferred here?
338 preferred = []
339 preferred_vars = []
340 pns = {}
341 for p in eligible:
342 pns[dataCache.pkg_fn[p]] = p
343 for p in eligible:
344 pn = dataCache.pkg_fn[p]
345 provides = dataCache.pn_provides[pn]
346 for provide in provides:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500347 prefervar = cfgData.getVar('PREFERRED_PROVIDER_%s' % provide)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500348 #logger.debug(1, "checking PREFERRED_PROVIDER_%s (value %s) against %s", provide, prefervar, pns.keys())
349 if prefervar in pns and pns[prefervar] not in preferred:
350 var = "PREFERRED_PROVIDER_%s = %s" % (provide, prefervar)
351 logger.verbose("selecting %s to satisfy runtime %s due to %s", prefervar, item, var)
352 preferred_vars.append(var)
353 pref = pns[prefervar]
354 eligible.remove(pref)
355 eligible = [pref] + eligible
356 preferred.append(pref)
357 break
358
359 numberPreferred = len(preferred)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500360
361 if numberPreferred > 1:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500362 logger.error("Trying to resolve runtime dependency %s resulted in conflicting PREFERRED_PROVIDER entries being found.\nThe providers found were: %s\nThe PREFERRED_PROVIDER entries resulting in this conflict were: %s. You could set PREFERRED_RPROVIDER_%s" % (item, preferred, preferred_vars, item))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500363
364 logger.debug(1, "sorted runtime providers for %s are: %s", item, eligible)
365
366 return eligible, numberPreferred
367
368regexp_cache = {}
369
370def getRuntimeProviders(dataCache, rdepend):
371 """
372 Return any providers of runtime dependency
373 """
374 rproviders = []
375
376 if rdepend in dataCache.rproviders:
377 rproviders += dataCache.rproviders[rdepend]
378
379 if rdepend in dataCache.packages:
380 rproviders += dataCache.packages[rdepend]
381
382 if rproviders:
383 return rproviders
384
385 # Only search dynamic packages if we can't find anything in other variables
386 for pattern in dataCache.packages_dynamic:
387 pattern = pattern.replace('+', "\+")
388 if pattern in regexp_cache:
389 regexp = regexp_cache[pattern]
390 else:
391 try:
392 regexp = re.compile(pattern)
393 except:
394 logger.error("Error parsing regular expression '%s'", pattern)
395 raise
396 regexp_cache[pattern] = regexp
397 if regexp.match(rdepend):
398 rproviders += dataCache.packages_dynamic[pattern]
399 logger.debug(1, "Assuming %s is a dynamic package, but it may not exist" % rdepend)
400
401 return rproviders
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500402
403
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600404def buildWorldTargetList(dataCache, task=None):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500405 """
406 Build package list for "bitbake world"
407 """
408 if dataCache.world_target:
409 return
410
411 logger.debug(1, "collating packages for \"world\"")
412 for f in dataCache.possible_world:
413 terminal = True
414 pn = dataCache.pkg_fn[f]
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600415 if task and task not in dataCache.task_deps[f]['tasks']:
416 logger.debug(2, "World build skipping %s as task %s doesn't exist", f, task)
417 terminal = False
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500418
419 for p in dataCache.pn_provides[pn]:
420 if p.startswith('virtual/'):
421 logger.debug(2, "World build skipping %s due to %s provider starting with virtual/", f, p)
422 terminal = False
423 break
424 for pf in dataCache.providers[p]:
425 if dataCache.pkg_fn[pf] != pn:
426 logger.debug(2, "World build skipping %s due to both us and %s providing %s", f, pf, p)
427 terminal = False
428 break
429 if terminal:
430 dataCache.world_target.add(pn)