blob: 7bfcdb84141ccf558fca2de266eb7e2204e5424b [file] [log] [blame]
"""
BitBake 'TaskData' implementation
Task data collection and handling
"""
# Copyright (C) 2006 Richard Purdie
#
# SPDX-License-Identifier: GPL-2.0-only
#
import logging
import re
import bb
logger = logging.getLogger("BitBake.TaskData")
def re_match_strings(target, strings):
"""
Whether or not the string 'target' matches
any one string of the strings which can be regular expression string
"""
for name in strings:
if name.startswith("^") or name.endswith("$"):
if re.match(name, target):
return True
elif name == target:
return True
return False
class TaskEntry:
def __init__(self):
self.tdepends = []
self.idepends = []
self.irdepends = []
class TaskData:
"""
BitBake Task Data implementation
"""
def __init__(self, abort = True, skiplist = None, allowincomplete = False):
self.build_targets = {}
self.run_targets = {}
self.external_targets = []
self.seenfns = []
self.taskentries = {}
self.depids = {}
self.rdepids = {}
self.consider_msgs_cache = []
self.failed_deps = []
self.failed_rdeps = []
self.failed_fns = []
self.abort = abort
self.allowincomplete = allowincomplete
self.skiplist = skiplist
self.mcdepends = []
def add_tasks(self, fn, dataCache):
"""
Add tasks for a given fn to the database
"""
task_deps = dataCache.task_deps[fn]
if fn in self.failed_fns:
bb.msg.fatal("TaskData", "Trying to re-add a failed file? Something is broken...")
# Check if we've already seen this fn
if fn in self.seenfns:
return
self.seenfns.append(fn)
self.add_extra_deps(fn, dataCache)
def add_mcdepends(task):
for dep in task_deps['mcdepends'][task].split():
if len(dep.split(':')) != 5:
bb.msg.fatal("TaskData", "Error for %s:%s[%s], multiconfig dependency %s does not contain exactly four ':' characters.\n Task '%s' should be specified in the form 'mc:fromMC:toMC:packagename:task'" % (fn, task, 'mcdepends', dep, 'mcdepends'))
if dep not in self.mcdepends:
self.mcdepends.append(dep)
# Common code for dep_name/depends = 'depends'/idepends and 'rdepends'/irdepends
def handle_deps(task, dep_name, depends, seen):
if dep_name in task_deps and task in task_deps[dep_name]:
ids = []
for dep in task_deps[dep_name][task].split():
if dep:
parts = dep.split(":")
if len(parts) != 2:
bb.msg.fatal("TaskData", "Error for %s:%s[%s], dependency %s in '%s' does not contain exactly one ':' character.\n Task '%s' should be specified in the form 'packagename:task'" % (fn, task, dep_name, dep, task_deps[dep_name][task], dep_name))
ids.append((parts[0], parts[1]))
seen(parts[0])
depends.extend(ids)
for task in task_deps['tasks']:
tid = "%s:%s" % (fn, task)
self.taskentries[tid] = TaskEntry()
# Work out task dependencies
parentids = []
for dep in task_deps['parents'][task]:
if dep not in task_deps['tasks']:
bb.debug(2, "Not adding dependency of %s on %s since %s does not exist" % (task, dep, dep))
continue
parentid = "%s:%s" % (fn, dep)
parentids.append(parentid)
self.taskentries[tid].tdepends.extend(parentids)
# Touch all intertask dependencies
handle_deps(task, 'depends', self.taskentries[tid].idepends, self.seen_build_target)
handle_deps(task, 'rdepends', self.taskentries[tid].irdepends, self.seen_run_target)
if 'mcdepends' in task_deps and task in task_deps['mcdepends']:
add_mcdepends(task)
# Work out build dependencies
if not fn in self.depids:
dependids = set()
for depend in dataCache.deps[fn]:
dependids.add(depend)
self.depids[fn] = list(dependids)
logger.debug2("Added dependencies %s for %s", str(dataCache.deps[fn]), fn)
# Work out runtime dependencies
if not fn in self.rdepids:
rdependids = set()
rdepends = dataCache.rundeps[fn]
rrecs = dataCache.runrecs[fn]
rdependlist = []
rreclist = []
for package in rdepends:
for rdepend in rdepends[package]:
rdependlist.append(rdepend)
rdependids.add(rdepend)
for package in rrecs:
for rdepend in rrecs[package]:
rreclist.append(rdepend)
rdependids.add(rdepend)
if rdependlist:
logger.debug2("Added runtime dependencies %s for %s", str(rdependlist), fn)
if rreclist:
logger.debug2("Added runtime recommendations %s for %s", str(rreclist), fn)
self.rdepids[fn] = list(rdependids)
for dep in self.depids[fn]:
self.seen_build_target(dep)
if dep in self.failed_deps:
self.fail_fn(fn)
return
for dep in self.rdepids[fn]:
self.seen_run_target(dep)
if dep in self.failed_rdeps:
self.fail_fn(fn)
return
def add_extra_deps(self, fn, dataCache):
func = dataCache.extradepsfunc.get(fn, None)
if func:
bb.providers.buildWorldTargetList(dataCache)
pn = dataCache.pkg_fn[fn]
params = {'deps': dataCache.deps[fn],
'world_target': dataCache.world_target,
'pkg_pn': dataCache.pkg_pn,
'self_pn': pn}
funcname = '_%s_calculate_extra_depends' % pn.replace('-', '_')
paramlist = ','.join(params.keys())
func = 'def %s(%s):\n%s\n\n%s(%s)' % (funcname, paramlist, func, funcname, paramlist)
bb.utils.better_exec(func, params)
def have_build_target(self, target):
"""
Have we a build target matching this name?
"""
if target in self.build_targets and self.build_targets[target]:
return True
return False
def have_runtime_target(self, target):
"""
Have we a runtime target matching this name?
"""
if target in self.run_targets and self.run_targets[target]:
return True
return False
def seen_build_target(self, name):
"""
Maintain a list of build targets
"""
if name not in self.build_targets:
self.build_targets[name] = []
def add_build_target(self, fn, item):
"""
Add a build target.
If already present, append the provider fn to the list
"""
if item in self.build_targets:
if fn in self.build_targets[item]:
return
self.build_targets[item].append(fn)
return
self.build_targets[item] = [fn]
def seen_run_target(self, name):
"""
Maintain a list of runtime build targets
"""
if name not in self.run_targets:
self.run_targets[name] = []
def add_runtime_target(self, fn, item):
"""
Add a runtime target.
If already present, append the provider fn to the list
"""
if item in self.run_targets:
if fn in self.run_targets[item]:
return
self.run_targets[item].append(fn)
return
self.run_targets[item] = [fn]
def mark_external_target(self, target):
"""
Mark a build target as being externally requested
"""
if target not in self.external_targets:
self.external_targets.append(target)
def get_unresolved_build_targets(self, dataCache):
"""
Return a list of build targets who's providers
are unknown.
"""
unresolved = []
for target in self.build_targets:
if re_match_strings(target, dataCache.ignored_dependencies):
continue
if target in self.failed_deps:
continue
if not self.build_targets[target]:
unresolved.append(target)
return unresolved
def get_unresolved_run_targets(self, dataCache):
"""
Return a list of runtime targets who's providers
are unknown.
"""
unresolved = []
for target in self.run_targets:
if re_match_strings(target, dataCache.ignored_dependencies):
continue
if target in self.failed_rdeps:
continue
if not self.run_targets[target]:
unresolved.append(target)
return unresolved
def get_provider(self, item):
"""
Return a list of providers of item
"""
return self.build_targets[item]
def get_dependees(self, item):
"""
Return a list of targets which depend on item
"""
dependees = []
for fn in self.depids:
if item in self.depids[fn]:
dependees.append(fn)
return dependees
def get_rdependees(self, item):
"""
Return a list of targets which depend on runtime item
"""
dependees = []
for fn in self.rdepids:
if item in self.rdepids[fn]:
dependees.append(fn)
return dependees
def get_reasons(self, item, runtime=False):
"""
Get the reason(s) for an item not being provided, if any
"""
reasons = []
if self.skiplist:
for fn in self.skiplist:
skipitem = self.skiplist[fn]
if skipitem.pn == item:
reasons.append("%s was skipped: %s" % (skipitem.pn, skipitem.skipreason))
elif runtime and item in skipitem.rprovides:
reasons.append("%s RPROVIDES %s but was skipped: %s" % (skipitem.pn, item, skipitem.skipreason))
elif not runtime and item in skipitem.provides:
reasons.append("%s PROVIDES %s but was skipped: %s" % (skipitem.pn, item, skipitem.skipreason))
return reasons
def get_close_matches(self, item, provider_list):
import difflib
if self.skiplist:
skipped = []
for fn in self.skiplist:
skipped.append(self.skiplist[fn].pn)
full_list = provider_list + skipped
else:
full_list = provider_list
return difflib.get_close_matches(item, full_list, cutoff=0.7)
def add_provider(self, cfgData, dataCache, item):
try:
self.add_provider_internal(cfgData, dataCache, item)
except bb.providers.NoProvider:
if self.abort:
raise
self.remove_buildtarget(item)
self.mark_external_target(item)
def add_provider_internal(self, cfgData, dataCache, item):
"""
Add the providers of item to the task data
Mark entries were specifically added externally as against dependencies
added internally during dependency resolution
"""
if re_match_strings(item, dataCache.ignored_dependencies):
return
if not item in dataCache.providers:
close_matches = self.get_close_matches(item, list(dataCache.providers.keys()))
# Is it in RuntimeProviders ?
all_p = bb.providers.getRuntimeProviders(dataCache, item)
for fn in all_p:
new = dataCache.pkg_fn[fn] + " RPROVIDES " + item
if new not in close_matches:
close_matches.append(new)
bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees(item), reasons=self.get_reasons(item), close_matches=close_matches), cfgData)
raise bb.providers.NoProvider(item)
if self.have_build_target(item):
return
all_p = dataCache.providers[item]
eligible, foundUnique = bb.providers.filterProviders(all_p, item, cfgData, dataCache)
eligible = [p for p in eligible if not p in self.failed_fns]
if not eligible:
bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees(item), reasons=["No eligible PROVIDERs exist for '%s'" % item]), cfgData)
raise bb.providers.NoProvider(item)
if len(eligible) > 1 and not foundUnique:
if item not in self.consider_msgs_cache:
providers_list = []
for fn in eligible:
providers_list.append(dataCache.pkg_fn[fn])
bb.event.fire(bb.event.MultipleProviders(item, providers_list), cfgData)
self.consider_msgs_cache.append(item)
for fn in eligible:
if fn in self.failed_fns:
continue
logger.debug2("adding %s to satisfy %s", fn, item)
self.add_build_target(fn, item)
self.add_tasks(fn, dataCache)
#item = dataCache.pkg_fn[fn]
def add_rprovider(self, cfgData, dataCache, item):
"""
Add the runtime providers of item to the task data
(takes item names from RDEPENDS/PACKAGES namespace)
"""
if re_match_strings(item, dataCache.ignored_dependencies):
return
if self.have_runtime_target(item):
return
all_p = bb.providers.getRuntimeProviders(dataCache, item)
if not all_p:
bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees(item), reasons=self.get_reasons(item, True)), cfgData)
raise bb.providers.NoRProvider(item)
eligible, numberPreferred = bb.providers.filterProvidersRunTime(all_p, item, cfgData, dataCache)
eligible = [p for p in eligible if not p in self.failed_fns]
if not eligible:
bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees(item), reasons=["No eligible RPROVIDERs exist for '%s'" % item]), cfgData)
raise bb.providers.NoRProvider(item)
if len(eligible) > 1 and numberPreferred == 0:
if item not in self.consider_msgs_cache:
providers_list = []
for fn in eligible:
providers_list.append(dataCache.pkg_fn[fn])
bb.event.fire(bb.event.MultipleProviders(item, providers_list, runtime=True), cfgData)
self.consider_msgs_cache.append(item)
if numberPreferred > 1:
if item not in self.consider_msgs_cache:
providers_list = []
for fn in eligible:
providers_list.append(dataCache.pkg_fn[fn])
bb.event.fire(bb.event.MultipleProviders(item, providers_list, runtime=True), cfgData)
self.consider_msgs_cache.append(item)
raise bb.providers.MultipleRProvider(item)
# run through the list until we find one that we can build
for fn in eligible:
if fn in self.failed_fns:
continue
logger.debug2("adding '%s' to satisfy runtime '%s'", fn, item)
self.add_runtime_target(fn, item)
self.add_tasks(fn, dataCache)
def fail_fn(self, fn, missing_list=None):
"""
Mark a file as failed (unbuildable)
Remove any references from build and runtime provider lists
missing_list, A list of missing requirements for this target
"""
if fn in self.failed_fns:
return
if not missing_list:
missing_list = []
logger.debug("File '%s' is unbuildable, removing...", fn)
self.failed_fns.append(fn)
for target in self.build_targets:
if fn in self.build_targets[target]:
self.build_targets[target].remove(fn)
if not self.build_targets[target]:
self.remove_buildtarget(target, missing_list)
for target in self.run_targets:
if fn in self.run_targets[target]:
self.run_targets[target].remove(fn)
if not self.run_targets[target]:
self.remove_runtarget(target, missing_list)
def remove_buildtarget(self, target, missing_list=None):
"""
Mark a build target as failed (unbuildable)
Trigger removal of any files that have this as a dependency
"""
if not missing_list:
missing_list = [target]
else:
missing_list = [target] + missing_list
logger.verbose("Target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain was: %s", target, missing_list)
self.failed_deps.append(target)
dependees = self.get_dependees(target)
for fn in dependees:
self.fail_fn(fn, missing_list)
for tid in self.taskentries:
for (idepend, idependtask) in self.taskentries[tid].idepends:
if idepend == target:
fn = tid.rsplit(":",1)[0]
self.fail_fn(fn, missing_list)
if self.abort and target in self.external_targets:
logger.error("Required build target '%s' has no buildable providers.\nMissing or unbuildable dependency chain was: %s", target, missing_list)
raise bb.providers.NoProvider(target)
def remove_runtarget(self, target, missing_list=None):
"""
Mark a run target as failed (unbuildable)
Trigger removal of any files that have this as a dependency
"""
if not missing_list:
missing_list = [target]
else:
missing_list = [target] + missing_list
logger.info("Runtime target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain was: %s", target, missing_list)
self.failed_rdeps.append(target)
dependees = self.get_rdependees(target)
for fn in dependees:
self.fail_fn(fn, missing_list)
for tid in self.taskentries:
for (idepend, idependtask) in self.taskentries[tid].irdepends:
if idepend == target:
fn = tid.rsplit(":",1)[0]
self.fail_fn(fn, missing_list)
def add_unresolved(self, cfgData, dataCache):
"""
Resolve all unresolved build and runtime targets
"""
logger.info("Resolving any missing task queue dependencies")
while True:
added = 0
for target in self.get_unresolved_build_targets(dataCache):
try:
self.add_provider_internal(cfgData, dataCache, target)
added = added + 1
except bb.providers.NoProvider:
if self.abort and target in self.external_targets and not self.allowincomplete:
raise
if not self.allowincomplete:
self.remove_buildtarget(target)
for target in self.get_unresolved_run_targets(dataCache):
try:
self.add_rprovider(cfgData, dataCache, target)
added = added + 1
except (bb.providers.NoRProvider, bb.providers.MultipleRProvider):
self.remove_runtarget(target)
logger.debug("Resolved " + str(added) + " extra dependencies")
if added == 0:
break
# self.dump_data()
def get_providermap(self, prefix=None):
provmap = {}
for name in self.build_targets:
if prefix and not name.startswith(prefix):
continue
if self.have_build_target(name):
provider = self.get_provider(name)
if provider:
provmap[name] = provider[0]
return provmap
def get_mcdepends(self):
return self.mcdepends
def dump_data(self):
"""
Dump some debug information on the internal data structures
"""
logger.debug3("build_names:")
logger.debug3(", ".join(self.build_targets))
logger.debug3("run_names:")
logger.debug3(", ".join(self.run_targets))
logger.debug3("build_targets:")
for target in self.build_targets:
targets = "None"
if target in self.build_targets:
targets = self.build_targets[target]
logger.debug3(" %s: %s", target, targets)
logger.debug3("run_targets:")
for target in self.run_targets:
targets = "None"
if target in self.run_targets:
targets = self.run_targets[target]
logger.debug3(" %s: %s", target, targets)
logger.debug3("tasks:")
for tid in self.taskentries:
logger.debug3(" %s: %s %s %s",
tid,
self.taskentries[tid].idepends,
self.taskentries[tid].irdepends,
self.taskentries[tid].tdepends)
logger.debug3("dependency ids (per fn):")
for fn in self.depids:
logger.debug3(" %s: %s", fn, self.depids[fn])
logger.debug3("runtime dependency ids (per fn):")
for fn in self.rdepids:
logger.debug3(" %s: %s", fn, self.rdepids[fn])