meta-openembedded and poky: subtree updates
Squash of the following due to dependencies among them
and OpenBMC changes:
meta-openembedded: subtree update:d0748372d2..9201611135
meta-openembedded: subtree update:9201611135..17fd382f34
poky: subtree update:9052e5b32a..2e11d97b6c
poky: subtree update:2e11d97b6c..a8544811d7
The change log was too large for the jenkins plugin
to handle therefore it has been removed. Here is
the first and last commit of each subtree:
meta-openembedded:d0748372d2
cppzmq: bump to version 4.6.0
meta-openembedded:17fd382f34
mpv: Remove X11 dependency
poky:9052e5b32a
package_ipk: Remove pointless comment to trigger rebuild
poky:a8544811d7
pbzip2: Fix license warning
Change-Id: If0fc6c37629642ee207a4ca2f7aa501a2c673cd6
Signed-off-by: Andrew Geissler <geissonator@yahoo.com>
diff --git a/poky/bitbake/lib/bb/runqueue.py b/poky/bitbake/lib/bb/runqueue.py
index 26492e7..16f076f 100644
--- a/poky/bitbake/lib/bb/runqueue.py
+++ b/poky/bitbake/lib/bb/runqueue.py
@@ -12,14 +12,12 @@
import copy
import os
import sys
-import signal
import stat
-import fcntl
import errno
import logging
import re
import bb
-from bb import msg, data, event
+from bb import msg, event
from bb import monitordisk
import subprocess
import pickle
@@ -29,6 +27,7 @@
bblogger = logging.getLogger("BitBake")
logger = logging.getLogger("BitBake.RunQueue")
+hashequiv_logger = logging.getLogger("BitBake.RunQueue.HashEquiv")
__find_sha256__ = re.compile( r'(?i)(?<![a-z0-9])[a-f0-9]{64}(?![a-z0-9])' )
@@ -148,8 +147,9 @@
"""
Return the id of the first task we find that is buildable
"""
+ # Once tasks are running we don't need to worry about them again
+ self.buildable.difference_update(self.rq.runq_running)
buildable = set(self.buildable)
- buildable.difference_update(self.rq.runq_running)
buildable.difference_update(self.rq.holdoff_tasks)
buildable.intersection_update(self.rq.tasks_covered | self.rq.tasks_notcovered)
if not buildable:
@@ -207,8 +207,6 @@
def newbuildable(self, task):
self.buildable.add(task)
- # Once tasks are running we don't need to worry about them again
- self.buildable.difference_update(self.rq.runq_running)
def removebuildable(self, task):
self.buildable.remove(task)
@@ -923,9 +921,11 @@
runq_build = {}
for task in self.cooker.configuration.runall:
+ if not task.startswith("do_"):
+ task = "do_{0}".format(task)
runall_tids = set()
for tid in list(self.runtaskentries):
- wanttid = fn_from_tid(tid) + ":do_%s" % task
+ wanttid = "{0}:{1}".format(fn_from_tid(tid), task)
if wanttid in delcount:
self.runtaskentries[wanttid] = delcount[wanttid]
if wanttid in self.runtaskentries:
@@ -952,7 +952,9 @@
runq_build = {}
for task in self.cooker.configuration.runonly:
- runonly_tids = { k: v for k, v in self.runtaskentries.items() if taskname_from_tid(k) == "do_%s" % task }
+ if not task.startswith("do_"):
+ task = "do_{0}".format(task)
+ runonly_tids = { k: v for k, v in self.runtaskentries.items() if taskname_from_tid(k) == task }
for tid in list(runonly_tids):
mark_active(tid,1)
@@ -1125,14 +1127,14 @@
self.init_progress_reporter.next_stage()
# Iterate over the task list looking for tasks with a 'setscene' function
- self.runq_setscene_tids = []
+ self.runq_setscene_tids = set()
if not self.cooker.configuration.nosetscene:
for tid in self.runtaskentries:
(mc, fn, taskname, _) = split_tid_mcfn(tid)
setscenetid = tid + "_setscene"
if setscenetid not in taskData[mc].taskentries:
continue
- self.runq_setscene_tids.append(tid)
+ self.runq_setscene_tids.add(tid)
self.init_progress_reporter.next_stage()
@@ -1182,10 +1184,8 @@
return len(self.runtaskentries)
def prepare_task_hash(self, tid):
- procdep = []
- for dep in self.runtaskentries[tid].depends:
- procdep.append(dep)
- self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(tid, procdep, self.dataCaches[mc_from_tid(tid)])
+ bb.parse.siggen.prep_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches[mc_from_tid(tid)])
+ self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches[mc_from_tid(tid)])
self.runtaskentries[tid].unihash = bb.parse.siggen.get_unihash(tid)
def dump_data(self):
@@ -1255,7 +1255,7 @@
"fakerootdirs" : self.rqdata.dataCaches[mc].fakerootdirs,
"fakerootnoenv" : self.rqdata.dataCaches[mc].fakerootnoenv,
"sigdata" : bb.parse.siggen.get_taskdata(),
- "logdefaultdebug" : bb.msg.loggerDefaultDebugLevel,
+ "logdefaultlevel" : bb.msg.loggerDefaultLogLevel,
"logdefaultverbose" : bb.msg.loggerDefaultVerbose,
"logdefaultverboselogs" : bb.msg.loggerVerboseLogs,
"logdefaultdomain" : bb.msg.loggerDefaultDomains,
@@ -1711,6 +1711,7 @@
self.runq_buildable = set()
self.runq_running = set()
self.runq_complete = set()
+ self.runq_tasksrun = set()
self.build_stamps = {}
self.build_stamps2 = []
@@ -1896,6 +1897,7 @@
self.stats.taskCompleted()
bb.event.fire(runQueueTaskCompleted(task, self.stats, self.rq), self.cfgData)
self.task_completeoutright(task)
+ self.runq_tasksrun.add(task)
def task_fail(self, task, exitcode):
"""
@@ -1962,12 +1964,17 @@
"""
self.rq.read_workers()
- self.process_possible_migrations()
+ if self.updated_taskhash_queue or self.pending_migrations:
+ self.process_possible_migrations()
+
+ if not hasattr(self, "sorted_setscene_tids"):
+ # Don't want to sort this set every execution
+ self.sorted_setscene_tids = sorted(self.rqdata.runq_setscene_tids)
task = None
if not self.sqdone and self.can_start_task():
# Find the next setscene to run
- for nexttask in sorted(self.rqdata.runq_setscene_tids):
+ for nexttask in self.sorted_setscene_tids:
if nexttask in self.sq_buildable and nexttask not in self.sq_running and self.sqdata.stamps[nexttask] not in self.build_stamps.values():
if nexttask not in self.sqdata.unskippable and len(self.sqdata.sq_revdeps[nexttask]) > 0 and self.sqdata.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and self.check_dependencies(nexttask, self.sqdata.sq_revdeps[nexttask]):
if nexttask not in self.rqdata.target_tids:
@@ -2052,7 +2059,7 @@
self.update_holdofftasks()
if not self.sq_live and not self.sqdone and not self.sq_deferred and not self.updated_taskhash_queue and not self.holdoff_tasks:
- logger.info("Setscene tasks completed")
+ hashequiv_logger.verbose("Setscene tasks completed")
err = self.summarise_scenequeue_errors()
if err:
@@ -2090,6 +2097,7 @@
logger.debug(2, "Stamp current task %s", task)
self.task_skip(task, "existing")
+ self.runq_tasksrun.add(task)
return True
taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn]
@@ -2248,6 +2256,7 @@
def process_possible_migrations(self):
changed = set()
+ toprocess = set()
for tid, unihash in self.updated_taskhash_queue.copy():
if tid in self.runq_running and tid not in self.runq_complete:
continue
@@ -2255,42 +2264,63 @@
self.updated_taskhash_queue.remove((tid, unihash))
if unihash != self.rqdata.runtaskentries[tid].unihash:
- logger.info("Task %s unihash changed to %s" % (tid, unihash))
+ hashequiv_logger.verbose("Task %s unihash changed to %s" % (tid, unihash))
self.rqdata.runtaskentries[tid].unihash = unihash
bb.parse.siggen.set_unihash(tid, unihash)
+ toprocess.add(tid)
- # Work out all tasks which depend on this one
- total = set()
- next = set(self.rqdata.runtaskentries[tid].revdeps)
- while next:
- current = next.copy()
- total = total |next
- next = set()
- for ntid in current:
- next |= self.rqdata.runtaskentries[ntid].revdeps
- next.difference_update(total)
+ # Work out all tasks which depend upon these
+ total = set()
+ next = set()
+ for p in toprocess:
+ next |= self.rqdata.runtaskentries[p].revdeps
+ while next:
+ current = next.copy()
+ total = total | next
+ next = set()
+ for ntid in current:
+ next |= self.rqdata.runtaskentries[ntid].revdeps
+ next.difference_update(total)
- # Now iterate those tasks in dependency order to regenerate their taskhash/unihash
- done = set()
- next = set(self.rqdata.runtaskentries[tid].revdeps)
- while next:
- current = next.copy()
- next = set()
- for tid in current:
- if not self.rqdata.runtaskentries[tid].depends.isdisjoint(total):
- continue
- procdep = []
- for dep in self.rqdata.runtaskentries[tid].depends:
- procdep.append(dep)
- orighash = self.rqdata.runtaskentries[tid].hash
- self.rqdata.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(tid, procdep, self.rqdata.dataCaches[mc_from_tid(tid)])
- origuni = self.rqdata.runtaskentries[tid].unihash
- self.rqdata.runtaskentries[tid].unihash = bb.parse.siggen.get_unihash(tid)
- logger.debug(1, "Task %s hash changes: %s->%s %s->%s" % (tid, orighash, self.rqdata.runtaskentries[tid].hash, origuni, self.rqdata.runtaskentries[tid].unihash))
- next |= self.rqdata.runtaskentries[tid].revdeps
- changed.add(tid)
- total.remove(tid)
- next.intersection_update(total)
+ # Now iterate those tasks in dependency order to regenerate their taskhash/unihash
+ next = set()
+ for p in total:
+ if len(self.rqdata.runtaskentries[p].depends) == 0:
+ next.add(p)
+ elif self.rqdata.runtaskentries[p].depends.isdisjoint(total):
+ next.add(p)
+
+ # When an item doesn't have dependencies in total, we can process it. Drop items from total when handled
+ while next:
+ current = next.copy()
+ next = set()
+ for tid in current:
+ if len(self.rqdata.runtaskentries[p].depends) and not self.rqdata.runtaskentries[tid].depends.isdisjoint(total):
+ continue
+ orighash = self.rqdata.runtaskentries[tid].hash
+ newhash = bb.parse.siggen.get_taskhash(tid, self.rqdata.runtaskentries[tid].depends, self.rqdata.dataCaches[mc_from_tid(tid)])
+ origuni = self.rqdata.runtaskentries[tid].unihash
+ newuni = bb.parse.siggen.get_unihash(tid)
+ # FIXME, need to check it can come from sstate at all for determinism?
+ remapped = False
+ if newuni == origuni:
+ # Nothing to do, we match, skip code below
+ remapped = True
+ elif tid in self.scenequeue_covered or tid in self.sq_live:
+ # Already ran this setscene task or it running. Report the new taskhash
+ bb.parse.siggen.report_unihash_equiv(tid, newhash, origuni, newuni, self.rqdata.dataCaches)
+ hashequiv_logger.verbose("Already covered setscene for %s so ignoring rehash (remap)" % (tid))
+ remapped = True
+
+ if not remapped:
+ #logger.debug(1, "Task %s hash changes: %s->%s %s->%s" % (tid, orighash, newhash, origuni, newuni))
+ self.rqdata.runtaskentries[tid].hash = newhash
+ self.rqdata.runtaskentries[tid].unihash = newuni
+ changed.add(tid)
+
+ next |= self.rqdata.runtaskentries[tid].revdeps
+ total.remove(tid)
+ next.intersection_update(total)
if changed:
for mc in self.rq.worker:
@@ -2298,7 +2328,7 @@
for mc in self.rq.fakeworker:
self.rq.fakeworker[mc].process.stdin.write(b"<newtaskhashes>" + pickle.dumps(bb.parse.siggen.get_taskhashes()) + b"</newtaskhashes>")
- logger.debug(1, pprint.pformat("Tasks changed:\n%s" % (changed)))
+ hashequiv_logger.debug(1, pprint.pformat("Tasks changed:\n%s" % (changed)))
for tid in changed:
if tid not in self.rqdata.runq_setscene_tids:
@@ -2306,24 +2336,18 @@
if tid not in self.pending_migrations:
self.pending_migrations.add(tid)
+ update_tasks = []
for tid in self.pending_migrations.copy():
- if tid in self.runq_running:
+ if tid in self.runq_running or tid in self.sq_live:
# Too late, task already running, not much we can do now
self.pending_migrations.remove(tid)
continue
- if tid in self.scenequeue_covered or tid in self.sq_live:
- # Already ran this setscene task or it running
- # Potentially risky, should we report this hash as a match?
- logger.info("Already covered setscene for %s so ignoring rehash" % (tid))
- self.pending_migrations.remove(tid)
- continue
-
valid = True
# Check no tasks this covers are running
for dep in self.sqdata.sq_covered_tasks[tid]:
if dep in self.runq_running and dep not in self.runq_complete:
- logger.debug(2, "Task %s is running which blocks setscene for %s from running" % (dep, tid))
+ hashequiv_logger.debug(2, "Task %s is running which blocks setscene for %s from running" % (dep, tid))
valid = False
break
if not valid:
@@ -2335,6 +2359,12 @@
if tid in self.tasks_scenequeue_done:
self.tasks_scenequeue_done.remove(tid)
for dep in self.sqdata.sq_covered_tasks[tid]:
+ if dep in self.runq_complete and dep not in self.runq_tasksrun:
+ bb.error("Task %s marked as completed but now needing to rerun? Aborting build." % dep)
+ self.failed_tids.append(tid)
+ self.rq.state = runQueueCleanUp
+ return
+
if dep not in self.runq_complete:
if dep in self.tasks_scenequeue_done and dep not in self.sqdata.unskippable:
self.tasks_scenequeue_done.remove(dep)
@@ -2343,7 +2373,12 @@
self.sq_buildable.remove(tid)
if tid in self.sq_running:
self.sq_running.remove(tid)
- if self.sqdata.sq_revdeps[tid].issubset(self.scenequeue_covered | self.scenequeue_notcovered):
+ harddepfail = False
+ for t in self.sqdata.sq_harddeps:
+ if tid in self.sqdata.sq_harddeps[t] and t in self.scenequeue_notcovered:
+ harddepfail = True
+ break
+ if not harddepfail and self.sqdata.sq_revdeps[tid].issubset(self.scenequeue_covered | self.scenequeue_notcovered):
if tid not in self.sq_buildable:
self.sq_buildable.add(tid)
if len(self.sqdata.sq_revdeps[tid]) == 0:
@@ -2367,13 +2402,17 @@
if tid in self.build_stamps:
del self.build_stamps[tid]
- origvalid = False
- if tid in self.sqdata.valid:
- origvalid = True
+ update_tasks.append((tid, harddepfail, tid in self.sqdata.valid))
+
+ if update_tasks:
self.sqdone = False
- update_scenequeue_data([tid], self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self, summary=False)
+ update_scenequeue_data([t[0] for t in update_tasks], self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self, summary=False)
+
+ for (tid, harddepfail, origvalid) in update_tasks:
if tid in self.sqdata.valid and not origvalid:
- logger.info("Setscene task %s became valid" % tid)
+ hashequiv_logger.verbose("Setscene task %s became valid" % tid)
+ if harddepfail:
+ self.sq_task_failoutright(tid)
if changed:
self.holdoff_need_update = True
@@ -2510,6 +2549,8 @@
msg = 'Task %s.%s attempted to execute unexpectedly and should have been setscened' % (pn, taskname)
else:
msg = 'Task %s.%s attempted to execute unexpectedly' % (pn, taskname)
+ for t in self.scenequeue_notcovered:
+ msg = msg + "\nTask %s, unihash %s, taskhash %s" % (t, self.rqdata.runtaskentries[t].unihash, self.rqdata.runtaskentries[t].hash)
logger.error(msg + '\nThis is usually due to missing setscene tasks. Those missing in this build were: %s' % pprint.pformat(self.scenequeue_notcovered))
return True
return False
@@ -2760,7 +2801,7 @@
sqdata.hashes[h] = tid
else:
sqrq.sq_deferred[tid] = sqdata.hashes[h]
- bb.warn("Deferring %s after %s" % (tid, sqdata.hashes[h]))
+ bb.note("Deferring %s after %s" % (tid, sqdata.hashes[h]))
class TaskFailure(Exception):