Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1 | """ |
| 2 | BitBake 'RunQueue' implementation |
| 3 | |
| 4 | Handles preparation and execution of a queue of tasks |
| 5 | """ |
| 6 | |
| 7 | # Copyright (C) 2006-2007 Richard Purdie |
| 8 | # |
Brad Bishop | c342db3 | 2019-05-15 21:57:59 -0400 | [diff] [blame] | 9 | # SPDX-License-Identifier: GPL-2.0-only |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 10 | # |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 11 | |
| 12 | import copy |
| 13 | import os |
| 14 | import sys |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 15 | import stat |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 16 | import errno |
| 17 | import logging |
| 18 | import re |
| 19 | import bb |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 20 | from bb import msg, event |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 21 | from bb import monitordisk |
| 22 | import subprocess |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 23 | import pickle |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 24 | from multiprocessing import Process |
Brad Bishop | 1932369 | 2019-04-05 15:28:33 -0400 | [diff] [blame] | 25 | import shlex |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 26 | import pprint |
Patrick Williams | db4c27e | 2022-08-05 08:10:29 -0500 | [diff] [blame] | 27 | import time |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 28 | |
| 29 | bblogger = logging.getLogger("BitBake") |
| 30 | logger = logging.getLogger("BitBake.RunQueue") |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 31 | hashequiv_logger = logging.getLogger("BitBake.RunQueue.HashEquiv") |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 32 | |
Brad Bishop | 1932369 | 2019-04-05 15:28:33 -0400 | [diff] [blame] | 33 | __find_sha256__ = re.compile( r'(?i)(?<![a-z0-9])[a-f0-9]{64}(?![a-z0-9])' ) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 34 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 35 | def fn_from_tid(tid): |
| 36 | return tid.rsplit(":", 1)[0] |
| 37 | |
| 38 | def taskname_from_tid(tid): |
| 39 | return tid.rsplit(":", 1)[1] |
| 40 | |
Andrew Geissler | 99467da | 2019-02-25 18:54:23 -0600 | [diff] [blame] | 41 | def mc_from_tid(tid): |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 42 | if tid.startswith('mc:') and tid.count(':') >= 2: |
Andrew Geissler | 99467da | 2019-02-25 18:54:23 -0600 | [diff] [blame] | 43 | return tid.split(':')[1] |
| 44 | return "" |
| 45 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 46 | def split_tid(tid): |
| 47 | (mc, fn, taskname, _) = split_tid_mcfn(tid) |
| 48 | return (mc, fn, taskname) |
| 49 | |
Andrew Geissler | 5a43b43 | 2020-06-13 10:46:56 -0500 | [diff] [blame] | 50 | def split_mc(n): |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 51 | if n.startswith("mc:") and n.count(':') >= 2: |
Andrew Geissler | 5a43b43 | 2020-06-13 10:46:56 -0500 | [diff] [blame] | 52 | _, mc, n = n.split(":", 2) |
| 53 | return (mc, n) |
| 54 | return ('', n) |
| 55 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 56 | def split_tid_mcfn(tid): |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 57 | if tid.startswith('mc:') and tid.count(':') >= 2: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 58 | elems = tid.split(':') |
| 59 | mc = elems[1] |
| 60 | fn = ":".join(elems[2:-1]) |
| 61 | taskname = elems[-1] |
Brad Bishop | 15ae250 | 2019-06-18 21:44:24 -0400 | [diff] [blame] | 62 | mcfn = "mc:" + mc + ":" + fn |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 63 | else: |
| 64 | tid = tid.rsplit(":", 1) |
| 65 | mc = "" |
| 66 | fn = tid[0] |
| 67 | taskname = tid[1] |
| 68 | mcfn = fn |
| 69 | |
| 70 | return (mc, fn, taskname, mcfn) |
| 71 | |
| 72 | def build_tid(mc, fn, taskname): |
| 73 | if mc: |
Brad Bishop | 15ae250 | 2019-06-18 21:44:24 -0400 | [diff] [blame] | 74 | return "mc:" + mc + ":" + fn + ":" + taskname |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 75 | return fn + ":" + taskname |
| 76 | |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 77 | # Index used to pair up potentially matching multiconfig tasks |
| 78 | # We match on PN, taskname and hash being equal |
| 79 | def pending_hash_index(tid, rqdata): |
| 80 | (mc, fn, taskname, taskfn) = split_tid_mcfn(tid) |
| 81 | pn = rqdata.dataCaches[mc].pkg_fn[taskfn] |
Brad Bishop | 00e122a | 2019-10-05 11:10:57 -0400 | [diff] [blame] | 82 | h = rqdata.runtaskentries[tid].unihash |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 83 | return pn + ":" + "taskname" + h |
| 84 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 85 | class RunQueueStats: |
| 86 | """ |
| 87 | Holds statistics on the tasks handled by the associated runQueue |
| 88 | """ |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 89 | def __init__(self, total, setscene_total): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 90 | self.completed = 0 |
| 91 | self.skipped = 0 |
| 92 | self.failed = 0 |
| 93 | self.active = 0 |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 94 | self.setscene_active = 0 |
| 95 | self.setscene_covered = 0 |
| 96 | self.setscene_notcovered = 0 |
| 97 | self.setscene_total = setscene_total |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 98 | self.total = total |
| 99 | |
| 100 | def copy(self): |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 101 | obj = self.__class__(self.total, self.setscene_total) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 102 | obj.__dict__.update(self.__dict__) |
| 103 | return obj |
| 104 | |
| 105 | def taskFailed(self): |
| 106 | self.active = self.active - 1 |
| 107 | self.failed = self.failed + 1 |
| 108 | |
Brad Bishop | 1a4b7ee | 2018-12-16 17:11:34 -0800 | [diff] [blame] | 109 | def taskCompleted(self): |
| 110 | self.active = self.active - 1 |
| 111 | self.completed = self.completed + 1 |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 112 | |
Brad Bishop | 1a4b7ee | 2018-12-16 17:11:34 -0800 | [diff] [blame] | 113 | def taskSkipped(self): |
| 114 | self.active = self.active + 1 |
| 115 | self.skipped = self.skipped + 1 |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 116 | |
| 117 | def taskActive(self): |
| 118 | self.active = self.active + 1 |
| 119 | |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 120 | def updateCovered(self, covered, notcovered): |
| 121 | self.setscene_covered = covered |
| 122 | self.setscene_notcovered = notcovered |
| 123 | |
| 124 | def updateActiveSetscene(self, active): |
| 125 | self.setscene_active = active |
| 126 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 127 | # These values indicate the next step due to be run in the |
| 128 | # runQueue state machine |
| 129 | runQueuePrepare = 2 |
| 130 | runQueueSceneInit = 3 |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 131 | runQueueRunning = 6 |
| 132 | runQueueFailed = 7 |
| 133 | runQueueCleanUp = 8 |
| 134 | runQueueComplete = 9 |
| 135 | |
| 136 | class RunQueueScheduler(object): |
| 137 | """ |
| 138 | Control the order tasks are scheduled in. |
| 139 | """ |
| 140 | name = "basic" |
| 141 | |
| 142 | def __init__(self, runqueue, rqdata): |
| 143 | """ |
| 144 | The default scheduler just returns the first buildable task (the |
| 145 | priority map is sorted by task number) |
| 146 | """ |
| 147 | self.rq = runqueue |
| 148 | self.rqdata = rqdata |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 149 | self.numTasks = len(self.rqdata.runtaskentries) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 150 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 151 | self.prio_map = [self.rqdata.runtaskentries.keys()] |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 152 | |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 153 | self.buildable = set() |
Brad Bishop | 1a4b7ee | 2018-12-16 17:11:34 -0800 | [diff] [blame] | 154 | self.skip_maxthread = {} |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 155 | self.stamps = {} |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 156 | for tid in self.rqdata.runtaskentries: |
| 157 | (mc, fn, taskname, taskfn) = split_tid_mcfn(tid) |
Andrew Geissler | 517393d | 2023-01-13 08:55:19 -0600 | [diff] [blame] | 158 | self.stamps[tid] = bb.parse.siggen.stampfile_mcfn(taskname, taskfn, extrainfo=False) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 159 | if tid in self.rq.runq_buildable: |
| 160 | self.buildable.append(tid) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 161 | |
| 162 | self.rev_prio_map = None |
Patrick Williams | db4c27e | 2022-08-05 08:10:29 -0500 | [diff] [blame] | 163 | self.is_pressure_usable() |
| 164 | |
| 165 | def is_pressure_usable(self): |
| 166 | """ |
| 167 | If monitoring pressure, return True if pressure files can be open and read. For example |
| 168 | openSUSE /proc/pressure/* files have readable file permissions but when read the error EOPNOTSUPP (Operation not supported) |
| 169 | is returned. |
| 170 | """ |
Patrick Williams | 92b42cb | 2022-09-03 06:53:57 -0500 | [diff] [blame] | 171 | if self.rq.max_cpu_pressure or self.rq.max_io_pressure or self.rq.max_memory_pressure: |
Patrick Williams | db4c27e | 2022-08-05 08:10:29 -0500 | [diff] [blame] | 172 | try: |
Patrick Williams | 92b42cb | 2022-09-03 06:53:57 -0500 | [diff] [blame] | 173 | with open("/proc/pressure/cpu") as cpu_pressure_fds, \ |
| 174 | open("/proc/pressure/io") as io_pressure_fds, \ |
| 175 | open("/proc/pressure/memory") as memory_pressure_fds: |
| 176 | |
Patrick Williams | db4c27e | 2022-08-05 08:10:29 -0500 | [diff] [blame] | 177 | self.prev_cpu_pressure = cpu_pressure_fds.readline().split()[4].split("=")[1] |
| 178 | self.prev_io_pressure = io_pressure_fds.readline().split()[4].split("=")[1] |
Patrick Williams | 92b42cb | 2022-09-03 06:53:57 -0500 | [diff] [blame] | 179 | self.prev_memory_pressure = memory_pressure_fds.readline().split()[4].split("=")[1] |
Patrick Williams | db4c27e | 2022-08-05 08:10:29 -0500 | [diff] [blame] | 180 | self.prev_pressure_time = time.time() |
| 181 | self.check_pressure = True |
| 182 | except: |
Patrick Williams | 92b42cb | 2022-09-03 06:53:57 -0500 | [diff] [blame] | 183 | bb.note("The /proc/pressure files can't be read. Continuing build without monitoring pressure") |
Patrick Williams | db4c27e | 2022-08-05 08:10:29 -0500 | [diff] [blame] | 184 | self.check_pressure = False |
| 185 | else: |
| 186 | self.check_pressure = False |
| 187 | |
| 188 | def exceeds_max_pressure(self): |
| 189 | """ |
| 190 | Monitor the difference in total pressure at least once per second, if |
Patrick Williams | 92b42cb | 2022-09-03 06:53:57 -0500 | [diff] [blame] | 191 | BB_PRESSURE_MAX_{CPU|IO|MEMORY} are set, return True if above threshold. |
Patrick Williams | db4c27e | 2022-08-05 08:10:29 -0500 | [diff] [blame] | 192 | """ |
| 193 | if self.check_pressure: |
Patrick Williams | 92b42cb | 2022-09-03 06:53:57 -0500 | [diff] [blame] | 194 | with open("/proc/pressure/cpu") as cpu_pressure_fds, \ |
| 195 | open("/proc/pressure/io") as io_pressure_fds, \ |
| 196 | open("/proc/pressure/memory") as memory_pressure_fds: |
Patrick Williams | db4c27e | 2022-08-05 08:10:29 -0500 | [diff] [blame] | 197 | # extract "total" from /proc/pressure/{cpu|io} |
| 198 | curr_cpu_pressure = cpu_pressure_fds.readline().split()[4].split("=")[1] |
| 199 | curr_io_pressure = io_pressure_fds.readline().split()[4].split("=")[1] |
Patrick Williams | 92b42cb | 2022-09-03 06:53:57 -0500 | [diff] [blame] | 200 | curr_memory_pressure = memory_pressure_fds.readline().split()[4].split("=")[1] |
Patrick Williams | db4c27e | 2022-08-05 08:10:29 -0500 | [diff] [blame] | 201 | exceeds_cpu_pressure = self.rq.max_cpu_pressure and (float(curr_cpu_pressure) - float(self.prev_cpu_pressure)) > self.rq.max_cpu_pressure |
| 202 | exceeds_io_pressure = self.rq.max_io_pressure and (float(curr_io_pressure) - float(self.prev_io_pressure)) > self.rq.max_io_pressure |
Patrick Williams | 92b42cb | 2022-09-03 06:53:57 -0500 | [diff] [blame] | 203 | exceeds_memory_pressure = self.rq.max_memory_pressure and (float(curr_memory_pressure) - float(self.prev_memory_pressure)) > self.rq.max_memory_pressure |
Patrick Williams | db4c27e | 2022-08-05 08:10:29 -0500 | [diff] [blame] | 204 | now = time.time() |
| 205 | if now - self.prev_pressure_time > 1.0: |
| 206 | self.prev_cpu_pressure = curr_cpu_pressure |
| 207 | self.prev_io_pressure = curr_io_pressure |
Patrick Williams | 92b42cb | 2022-09-03 06:53:57 -0500 | [diff] [blame] | 208 | self.prev_memory_pressure = curr_memory_pressure |
Patrick Williams | db4c27e | 2022-08-05 08:10:29 -0500 | [diff] [blame] | 209 | self.prev_pressure_time = now |
Patrick Williams | 92b42cb | 2022-09-03 06:53:57 -0500 | [diff] [blame] | 210 | return (exceeds_cpu_pressure or exceeds_io_pressure or exceeds_memory_pressure) |
Patrick Williams | db4c27e | 2022-08-05 08:10:29 -0500 | [diff] [blame] | 211 | return False |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 212 | |
| 213 | def next_buildable_task(self): |
| 214 | """ |
| 215 | Return the id of the first task we find that is buildable |
| 216 | """ |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 217 | # Once tasks are running we don't need to worry about them again |
| 218 | self.buildable.difference_update(self.rq.runq_running) |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 219 | buildable = set(self.buildable) |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 220 | buildable.difference_update(self.rq.holdoff_tasks) |
| 221 | buildable.intersection_update(self.rq.tasks_covered | self.rq.tasks_notcovered) |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 222 | if not buildable: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 223 | return None |
Brad Bishop | 1a4b7ee | 2018-12-16 17:11:34 -0800 | [diff] [blame] | 224 | |
Patrick Williams | db4c27e | 2022-08-05 08:10:29 -0500 | [diff] [blame] | 225 | # Bitbake requires that at least one task be active. Only check for pressure if |
| 226 | # this is the case, otherwise the pressure limitation could result in no tasks |
| 227 | # being active and no new tasks started thereby, at times, breaking the scheduler. |
| 228 | if self.rq.stats.active and self.exceeds_max_pressure(): |
| 229 | return None |
| 230 | |
Brad Bishop | 1a4b7ee | 2018-12-16 17:11:34 -0800 | [diff] [blame] | 231 | # Filter out tasks that have a max number of threads that have been exceeded |
| 232 | skip_buildable = {} |
| 233 | for running in self.rq.runq_running.difference(self.rq.runq_complete): |
| 234 | rtaskname = taskname_from_tid(running) |
| 235 | if rtaskname not in self.skip_maxthread: |
| 236 | self.skip_maxthread[rtaskname] = self.rq.cfgData.getVarFlag(rtaskname, "number_threads") |
| 237 | if not self.skip_maxthread[rtaskname]: |
| 238 | continue |
| 239 | if rtaskname in skip_buildable: |
| 240 | skip_buildable[rtaskname] += 1 |
| 241 | else: |
| 242 | skip_buildable[rtaskname] = 1 |
| 243 | |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 244 | if len(buildable) == 1: |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 245 | tid = buildable.pop() |
Brad Bishop | 1a4b7ee | 2018-12-16 17:11:34 -0800 | [diff] [blame] | 246 | taskname = taskname_from_tid(tid) |
| 247 | if taskname in skip_buildable and skip_buildable[taskname] >= int(self.skip_maxthread[taskname]): |
| 248 | return None |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 249 | stamp = self.stamps[tid] |
| 250 | if stamp not in self.rq.build_stamps.values(): |
| 251 | return tid |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 252 | |
| 253 | if not self.rev_prio_map: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 254 | self.rev_prio_map = {} |
| 255 | for tid in self.rqdata.runtaskentries: |
| 256 | self.rev_prio_map[tid] = self.prio_map.index(tid) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 257 | |
| 258 | best = None |
| 259 | bestprio = None |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 260 | for tid in buildable: |
Brad Bishop | 1a4b7ee | 2018-12-16 17:11:34 -0800 | [diff] [blame] | 261 | taskname = taskname_from_tid(tid) |
| 262 | if taskname in skip_buildable and skip_buildable[taskname] >= int(self.skip_maxthread[taskname]): |
| 263 | continue |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 264 | prio = self.rev_prio_map[tid] |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 265 | if bestprio is None or bestprio > prio: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 266 | stamp = self.stamps[tid] |
| 267 | if stamp in self.rq.build_stamps.values(): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 268 | continue |
| 269 | bestprio = prio |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 270 | best = tid |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 271 | |
| 272 | return best |
| 273 | |
| 274 | def next(self): |
| 275 | """ |
| 276 | Return the id of the task we should build next |
| 277 | """ |
Brad Bishop | 1a4b7ee | 2018-12-16 17:11:34 -0800 | [diff] [blame] | 278 | if self.rq.can_start_task(): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 279 | return self.next_buildable_task() |
| 280 | |
Brad Bishop | 316dfdd | 2018-06-25 12:45:53 -0400 | [diff] [blame] | 281 | def newbuildable(self, task): |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 282 | self.buildable.add(task) |
| 283 | |
| 284 | def removebuildable(self, task): |
| 285 | self.buildable.remove(task) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 286 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 287 | def describe_task(self, taskid): |
| 288 | result = 'ID %s' % taskid |
| 289 | if self.rev_prio_map: |
| 290 | result = result + (' pri %d' % self.rev_prio_map[taskid]) |
| 291 | return result |
| 292 | |
| 293 | def dump_prio(self, comment): |
| 294 | bb.debug(3, '%s (most important first):\n%s' % |
| 295 | (comment, |
| 296 | '\n'.join(['%d. %s' % (index + 1, self.describe_task(taskid)) for |
| 297 | index, taskid in enumerate(self.prio_map)]))) |
| 298 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 299 | class RunQueueSchedulerSpeed(RunQueueScheduler): |
| 300 | """ |
| 301 | A scheduler optimised for speed. The priority map is sorted by task weight, |
| 302 | heavier weighted tasks (tasks needed by the most other tasks) are run first. |
| 303 | """ |
| 304 | name = "speed" |
| 305 | |
| 306 | def __init__(self, runqueue, rqdata): |
| 307 | """ |
| 308 | The priority map is sorted by task weight. |
| 309 | """ |
| 310 | RunQueueScheduler.__init__(self, runqueue, rqdata) |
| 311 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 312 | weights = {} |
| 313 | for tid in self.rqdata.runtaskentries: |
| 314 | weight = self.rqdata.runtaskentries[tid].weight |
| 315 | if not weight in weights: |
| 316 | weights[weight] = [] |
| 317 | weights[weight].append(tid) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 318 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 319 | self.prio_map = [] |
| 320 | for weight in sorted(weights): |
| 321 | for w in weights[weight]: |
| 322 | self.prio_map.append(w) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 323 | |
| 324 | self.prio_map.reverse() |
| 325 | |
| 326 | class RunQueueSchedulerCompletion(RunQueueSchedulerSpeed): |
| 327 | """ |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 328 | A scheduler optimised to complete .bb files as quickly as possible. The |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 329 | priority map is sorted by task weight, but then reordered so once a given |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 330 | .bb file starts to build, it's completed as quickly as possible by |
| 331 | running all tasks related to the same .bb file one after the after. |
| 332 | This works well where disk space is at a premium and classes like OE's |
| 333 | rm_work are in force. |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 334 | """ |
| 335 | name = "completion" |
| 336 | |
| 337 | def __init__(self, runqueue, rqdata): |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 338 | super(RunQueueSchedulerCompletion, self).__init__(runqueue, rqdata) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 339 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 340 | # Extract list of tasks for each recipe, with tasks sorted |
| 341 | # ascending from "must run first" (typically do_fetch) to |
| 342 | # "runs last" (do_build). The speed scheduler prioritizes |
| 343 | # tasks that must run first before the ones that run later; |
| 344 | # this is what we depend on here. |
| 345 | task_lists = {} |
| 346 | for taskid in self.prio_map: |
| 347 | fn, taskname = taskid.rsplit(':', 1) |
| 348 | task_lists.setdefault(fn, []).append(taskname) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 349 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 350 | # Now unify the different task lists. The strategy is that |
| 351 | # common tasks get skipped and new ones get inserted after the |
| 352 | # preceeding common one(s) as they are found. Because task |
| 353 | # lists should differ only by their number of tasks, but not |
| 354 | # the ordering of the common tasks, this should result in a |
| 355 | # deterministic result that is a superset of the individual |
| 356 | # task ordering. |
| 357 | all_tasks = [] |
| 358 | for recipe, new_tasks in task_lists.items(): |
| 359 | index = 0 |
| 360 | old_task = all_tasks[index] if index < len(all_tasks) else None |
| 361 | for new_task in new_tasks: |
| 362 | if old_task == new_task: |
| 363 | # Common task, skip it. This is the fast-path which |
| 364 | # avoids a full search. |
| 365 | index += 1 |
| 366 | old_task = all_tasks[index] if index < len(all_tasks) else None |
| 367 | else: |
| 368 | try: |
| 369 | index = all_tasks.index(new_task) |
| 370 | # Already present, just not at the current |
| 371 | # place. We re-synchronized by changing the |
| 372 | # index so that it matches again. Now |
| 373 | # move on to the next existing task. |
| 374 | index += 1 |
| 375 | old_task = all_tasks[index] if index < len(all_tasks) else None |
| 376 | except ValueError: |
| 377 | # Not present. Insert before old_task, which |
| 378 | # remains the same (but gets shifted back). |
| 379 | all_tasks.insert(index, new_task) |
| 380 | index += 1 |
| 381 | bb.debug(3, 'merged task list: %s' % all_tasks) |
| 382 | |
| 383 | # Now reverse the order so that tasks that finish the work on one |
| 384 | # recipe are considered more imporant (= come first). The ordering |
| 385 | # is now so that do_build is most important. |
| 386 | all_tasks.reverse() |
| 387 | |
| 388 | # Group tasks of the same kind before tasks of less important |
| 389 | # kinds at the head of the queue (because earlier = lower |
| 390 | # priority number = runs earlier), while preserving the |
| 391 | # ordering by recipe. If recipe foo is more important than |
| 392 | # bar, then the goal is to work on foo's do_populate_sysroot |
| 393 | # before bar's do_populate_sysroot and on the more important |
| 394 | # tasks of foo before any of the less important tasks in any |
| 395 | # other recipe (if those other recipes are more important than |
| 396 | # foo). |
| 397 | # |
| 398 | # All of this only applies when tasks are runable. Explicit |
| 399 | # dependencies still override this ordering by priority. |
| 400 | # |
| 401 | # Here's an example why this priority re-ordering helps with |
| 402 | # minimizing disk usage. Consider a recipe foo with a higher |
| 403 | # priority than bar where foo DEPENDS on bar. Then the |
| 404 | # implicit rule (from base.bbclass) is that foo's do_configure |
| 405 | # depends on bar's do_populate_sysroot. This ensures that |
| 406 | # bar's do_populate_sysroot gets done first. Normally the |
| 407 | # tasks from foo would continue to run once that is done, and |
| 408 | # bar only gets completed and cleaned up later. By ordering |
| 409 | # bar's task that depend on bar's do_populate_sysroot before foo's |
| 410 | # do_configure, that problem gets avoided. |
| 411 | task_index = 0 |
| 412 | self.dump_prio('original priorities') |
| 413 | for task in all_tasks: |
| 414 | for index in range(task_index, self.numTasks): |
| 415 | taskid = self.prio_map[index] |
| 416 | taskname = taskid.rsplit(':', 1)[1] |
| 417 | if taskname == task: |
| 418 | del self.prio_map[index] |
| 419 | self.prio_map.insert(task_index, taskid) |
| 420 | task_index += 1 |
| 421 | self.dump_prio('completion priorities') |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 422 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 423 | class RunTaskEntry(object): |
| 424 | def __init__(self): |
| 425 | self.depends = set() |
| 426 | self.revdeps = set() |
| 427 | self.hash = None |
Brad Bishop | 1932369 | 2019-04-05 15:28:33 -0400 | [diff] [blame] | 428 | self.unihash = None |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 429 | self.task = None |
| 430 | self.weight = 1 |
| 431 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 432 | class RunQueueData: |
| 433 | """ |
| 434 | BitBake Run Queue implementation |
| 435 | """ |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 436 | def __init__(self, rq, cooker, cfgData, dataCaches, taskData, targets): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 437 | self.cooker = cooker |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 438 | self.dataCaches = dataCaches |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 439 | self.taskData = taskData |
| 440 | self.targets = targets |
| 441 | self.rq = rq |
| 442 | self.warn_multi_bb = False |
| 443 | |
Andrew Geissler | 7e0e3c0 | 2022-02-25 20:34:39 +0000 | [diff] [blame] | 444 | self.multi_provider_allowed = (cfgData.getVar("BB_MULTI_PROVIDER_ALLOWED") or "").split() |
| 445 | self.setscene_ignore_tasks = get_setscene_enforce_ignore_tasks(cfgData, targets) |
| 446 | self.setscene_ignore_tasks_checked = False |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 447 | self.setscene_enforce = (cfgData.getVar('BB_SETSCENE_ENFORCE') == "1") |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 448 | self.init_progress_reporter = bb.progress.DummyMultiStageProcessProgressReporter() |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 449 | |
| 450 | self.reset() |
| 451 | |
| 452 | def reset(self): |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 453 | self.runtaskentries = {} |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 454 | |
| 455 | def runq_depends_names(self, ids): |
| 456 | import re |
| 457 | ret = [] |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 458 | for id in ids: |
| 459 | nam = os.path.basename(id) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 460 | nam = re.sub("_[^,]*,", ",", nam) |
| 461 | ret.extend([nam]) |
| 462 | return ret |
| 463 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 464 | def get_task_hash(self, tid): |
| 465 | return self.runtaskentries[tid].hash |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 466 | |
Brad Bishop | 1932369 | 2019-04-05 15:28:33 -0400 | [diff] [blame] | 467 | def get_task_unihash(self, tid): |
| 468 | return self.runtaskentries[tid].unihash |
| 469 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 470 | def get_user_idstring(self, tid, task_name_suffix = ""): |
| 471 | return tid + task_name_suffix |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 472 | |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 473 | def get_short_user_idstring(self, task, task_name_suffix = ""): |
Brad Bishop | 37a0e4d | 2017-12-04 01:01:44 -0500 | [diff] [blame] | 474 | (mc, fn, taskname, taskfn) = split_tid_mcfn(task) |
| 475 | pn = self.dataCaches[mc].pkg_fn[taskfn] |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 476 | taskname = taskname_from_tid(task) + task_name_suffix |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 477 | return "%s:%s" % (pn, taskname) |
| 478 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 479 | def circular_depchains_handler(self, tasks): |
| 480 | """ |
| 481 | Some tasks aren't buildable, likely due to circular dependency issues. |
| 482 | Identify the circular dependencies and print them in a user readable format. |
| 483 | """ |
| 484 | from copy import deepcopy |
| 485 | |
| 486 | valid_chains = [] |
| 487 | explored_deps = {} |
| 488 | msgs = [] |
| 489 | |
Andrew Geissler | 99467da | 2019-02-25 18:54:23 -0600 | [diff] [blame] | 490 | class TooManyLoops(Exception): |
| 491 | pass |
| 492 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 493 | def chain_reorder(chain): |
| 494 | """ |
| 495 | Reorder a dependency chain so the lowest task id is first |
| 496 | """ |
| 497 | lowest = 0 |
| 498 | new_chain = [] |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 499 | for entry in range(len(chain)): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 500 | if chain[entry] < chain[lowest]: |
| 501 | lowest = entry |
| 502 | new_chain.extend(chain[lowest:]) |
| 503 | new_chain.extend(chain[:lowest]) |
| 504 | return new_chain |
| 505 | |
| 506 | def chain_compare_equal(chain1, chain2): |
| 507 | """ |
| 508 | Compare two dependency chains and see if they're the same |
| 509 | """ |
| 510 | if len(chain1) != len(chain2): |
| 511 | return False |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 512 | for index in range(len(chain1)): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 513 | if chain1[index] != chain2[index]: |
| 514 | return False |
| 515 | return True |
| 516 | |
| 517 | def chain_array_contains(chain, chain_array): |
| 518 | """ |
| 519 | Return True if chain_array contains chain |
| 520 | """ |
| 521 | for ch in chain_array: |
| 522 | if chain_compare_equal(ch, chain): |
| 523 | return True |
| 524 | return False |
| 525 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 526 | def find_chains(tid, prev_chain): |
| 527 | prev_chain.append(tid) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 528 | total_deps = [] |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 529 | total_deps.extend(self.runtaskentries[tid].revdeps) |
| 530 | for revdep in self.runtaskentries[tid].revdeps: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 531 | if revdep in prev_chain: |
| 532 | idx = prev_chain.index(revdep) |
| 533 | # To prevent duplicates, reorder the chain to start with the lowest taskid |
| 534 | # and search through an array of those we've already printed |
| 535 | chain = prev_chain[idx:] |
| 536 | new_chain = chain_reorder(chain) |
| 537 | if not chain_array_contains(new_chain, valid_chains): |
| 538 | valid_chains.append(new_chain) |
| 539 | msgs.append("Dependency loop #%d found:\n" % len(valid_chains)) |
| 540 | for dep in new_chain: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 541 | msgs.append(" Task %s (dependent Tasks %s)\n" % (dep, self.runq_depends_names(self.runtaskentries[dep].depends))) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 542 | msgs.append("\n") |
| 543 | if len(valid_chains) > 10: |
Andrew Geissler | 7e0e3c0 | 2022-02-25 20:34:39 +0000 | [diff] [blame] | 544 | msgs.append("Halted dependency loops search after 10 matches.\n") |
Andrew Geissler | 99467da | 2019-02-25 18:54:23 -0600 | [diff] [blame] | 545 | raise TooManyLoops |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 546 | continue |
| 547 | scan = False |
| 548 | if revdep not in explored_deps: |
| 549 | scan = True |
| 550 | elif revdep in explored_deps[revdep]: |
| 551 | scan = True |
| 552 | else: |
| 553 | for dep in prev_chain: |
| 554 | if dep in explored_deps[revdep]: |
| 555 | scan = True |
| 556 | if scan: |
| 557 | find_chains(revdep, copy.deepcopy(prev_chain)) |
| 558 | for dep in explored_deps[revdep]: |
| 559 | if dep not in total_deps: |
| 560 | total_deps.append(dep) |
| 561 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 562 | explored_deps[tid] = total_deps |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 563 | |
Andrew Geissler | 99467da | 2019-02-25 18:54:23 -0600 | [diff] [blame] | 564 | try: |
| 565 | for task in tasks: |
| 566 | find_chains(task, []) |
| 567 | except TooManyLoops: |
| 568 | pass |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 569 | |
| 570 | return msgs |
| 571 | |
| 572 | def calculate_task_weights(self, endpoints): |
| 573 | """ |
| 574 | Calculate a number representing the "weight" of each task. Heavier weighted tasks |
| 575 | have more dependencies and hence should be executed sooner for maximum speed. |
| 576 | |
| 577 | This function also sanity checks the task list finding tasks that are not |
| 578 | possible to execute due to circular dependencies. |
| 579 | """ |
| 580 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 581 | numTasks = len(self.runtaskentries) |
| 582 | weight = {} |
| 583 | deps_left = {} |
| 584 | task_done = {} |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 585 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 586 | for tid in self.runtaskentries: |
| 587 | task_done[tid] = False |
| 588 | weight[tid] = 1 |
| 589 | deps_left[tid] = len(self.runtaskentries[tid].revdeps) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 590 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 591 | for tid in endpoints: |
| 592 | weight[tid] = 10 |
| 593 | task_done[tid] = True |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 594 | |
| 595 | while True: |
| 596 | next_points = [] |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 597 | for tid in endpoints: |
| 598 | for revdep in self.runtaskentries[tid].depends: |
| 599 | weight[revdep] = weight[revdep] + weight[tid] |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 600 | deps_left[revdep] = deps_left[revdep] - 1 |
| 601 | if deps_left[revdep] == 0: |
| 602 | next_points.append(revdep) |
| 603 | task_done[revdep] = True |
| 604 | endpoints = next_points |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 605 | if not next_points: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 606 | break |
| 607 | |
| 608 | # Circular dependency sanity check |
| 609 | problem_tasks = [] |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 610 | for tid in self.runtaskentries: |
| 611 | if task_done[tid] is False or deps_left[tid] != 0: |
| 612 | problem_tasks.append(tid) |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 613 | logger.debug2("Task %s is not buildable", tid) |
| 614 | logger.debug2("(Complete marker was %s and the remaining dependency count was %s)\n", task_done[tid], deps_left[tid]) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 615 | self.runtaskentries[tid].weight = weight[tid] |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 616 | |
| 617 | if problem_tasks: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 618 | message = "%s unbuildable tasks were found.\n" % len(problem_tasks) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 619 | message = message + "These are usually caused by circular dependencies and any circular dependency chains found will be printed below. Increase the debug level to see a list of unbuildable tasks.\n\n" |
| 620 | message = message + "Identifying dependency loops (this may take a short while)...\n" |
| 621 | logger.error(message) |
| 622 | |
| 623 | msgs = self.circular_depchains_handler(problem_tasks) |
| 624 | |
| 625 | message = "\n" |
| 626 | for msg in msgs: |
| 627 | message = message + msg |
| 628 | bb.msg.fatal("RunQueue", message) |
| 629 | |
| 630 | return weight |
| 631 | |
| 632 | def prepare(self): |
| 633 | """ |
| 634 | Turn a set of taskData into a RunQueue and compute data needed |
| 635 | to optimise the execution order. |
| 636 | """ |
| 637 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 638 | runq_build = {} |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 639 | recursivetasks = {} |
| 640 | recursiveitasks = {} |
| 641 | recursivetasksselfref = set() |
| 642 | |
| 643 | taskData = self.taskData |
| 644 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 645 | found = False |
| 646 | for mc in self.taskData: |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 647 | if taskData[mc].taskentries: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 648 | found = True |
| 649 | break |
| 650 | if not found: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 651 | # Nothing to do |
| 652 | return 0 |
| 653 | |
Andrew Geissler | 517393d | 2023-01-13 08:55:19 -0600 | [diff] [blame] | 654 | bb.parse.siggen.setup_datacache(self.dataCaches) |
| 655 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 656 | self.init_progress_reporter.start() |
| 657 | self.init_progress_reporter.next_stage() |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 658 | |
| 659 | # Step A - Work out a list of tasks to run |
| 660 | # |
| 661 | # Taskdata gives us a list of possible providers for every build and run |
| 662 | # target ordered by priority. It also gives information on each of those |
| 663 | # providers. |
| 664 | # |
| 665 | # To create the actual list of tasks to execute we fix the list of |
| 666 | # providers and then resolve the dependencies into task IDs. This |
| 667 | # process is repeated for each type of dependency (tdepends, deptask, |
| 668 | # rdeptast, recrdeptask, idepends). |
| 669 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 670 | def add_build_dependencies(depids, tasknames, depends, mc): |
| 671 | for depname in depids: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 672 | # Won't be in build_targets if ASSUME_PROVIDED |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 673 | if depname not in taskData[mc].build_targets or not taskData[mc].build_targets[depname]: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 674 | continue |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 675 | depdata = taskData[mc].build_targets[depname][0] |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 676 | if depdata is None: |
| 677 | continue |
| 678 | for taskname in tasknames: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 679 | t = depdata + ":" + taskname |
| 680 | if t in taskData[mc].taskentries: |
| 681 | depends.add(t) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 682 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 683 | def add_runtime_dependencies(depids, tasknames, depends, mc): |
| 684 | for depname in depids: |
| 685 | if depname not in taskData[mc].run_targets or not taskData[mc].run_targets[depname]: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 686 | continue |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 687 | depdata = taskData[mc].run_targets[depname][0] |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 688 | if depdata is None: |
| 689 | continue |
| 690 | for taskname in tasknames: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 691 | t = depdata + ":" + taskname |
| 692 | if t in taskData[mc].taskentries: |
| 693 | depends.add(t) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 694 | |
Brad Bishop | 1a4b7ee | 2018-12-16 17:11:34 -0800 | [diff] [blame] | 695 | def add_mc_dependencies(mc, tid): |
| 696 | mcdeps = taskData[mc].get_mcdepends() |
| 697 | for dep in mcdeps: |
| 698 | mcdependency = dep.split(':') |
| 699 | pn = mcdependency[3] |
| 700 | frommc = mcdependency[1] |
| 701 | mcdep = mcdependency[2] |
| 702 | deptask = mcdependency[4] |
Andrew Geissler | 517393d | 2023-01-13 08:55:19 -0600 | [diff] [blame] | 703 | if mcdep not in taskData: |
| 704 | bb.fatal("Multiconfig '%s' is referenced in multiconfig dependency '%s' but not enabled in BBMULTICONFIG?" % (mcdep, dep)) |
Brad Bishop | 1a4b7ee | 2018-12-16 17:11:34 -0800 | [diff] [blame] | 705 | if mc == frommc: |
| 706 | fn = taskData[mcdep].build_targets[pn][0] |
| 707 | newdep = '%s:%s' % (fn,deptask) |
| 708 | taskData[mc].taskentries[tid].tdepends.append(newdep) |
| 709 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 710 | for mc in taskData: |
| 711 | for tid in taskData[mc].taskentries: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 712 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 713 | (mc, fn, taskname, taskfn) = split_tid_mcfn(tid) |
| 714 | #runtid = build_tid(mc, fn, taskname) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 715 | |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 716 | #logger.debug2("Processing %s,%s:%s", mc, fn, taskname) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 717 | |
| 718 | depends = set() |
| 719 | task_deps = self.dataCaches[mc].task_deps[taskfn] |
| 720 | |
| 721 | self.runtaskentries[tid] = RunTaskEntry() |
| 722 | |
| 723 | if fn in taskData[mc].failed_fns: |
| 724 | continue |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 725 | |
Brad Bishop | 1a4b7ee | 2018-12-16 17:11:34 -0800 | [diff] [blame] | 726 | # We add multiconfig dependencies before processing internal task deps (tdepends) |
| 727 | if 'mcdepends' in task_deps and taskname in task_deps['mcdepends']: |
| 728 | add_mc_dependencies(mc, tid) |
| 729 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 730 | # Resolve task internal dependencies |
| 731 | # |
| 732 | # e.g. addtask before X after Y |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 733 | for t in taskData[mc].taskentries[tid].tdepends: |
Brad Bishop | 1a4b7ee | 2018-12-16 17:11:34 -0800 | [diff] [blame] | 734 | (depmc, depfn, deptaskname, _) = split_tid_mcfn(t) |
| 735 | depends.add(build_tid(depmc, depfn, deptaskname)) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 736 | |
| 737 | # Resolve 'deptask' dependencies |
| 738 | # |
| 739 | # e.g. do_sometask[deptask] = "do_someothertask" |
| 740 | # (makes sure sometask runs after someothertask of all DEPENDS) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 741 | if 'deptask' in task_deps and taskname in task_deps['deptask']: |
| 742 | tasknames = task_deps['deptask'][taskname].split() |
| 743 | add_build_dependencies(taskData[mc].depids[taskfn], tasknames, depends, mc) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 744 | |
| 745 | # Resolve 'rdeptask' dependencies |
| 746 | # |
| 747 | # e.g. do_sometask[rdeptask] = "do_someothertask" |
| 748 | # (makes sure sometask runs after someothertask of all RDEPENDS) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 749 | if 'rdeptask' in task_deps and taskname in task_deps['rdeptask']: |
| 750 | tasknames = task_deps['rdeptask'][taskname].split() |
| 751 | add_runtime_dependencies(taskData[mc].rdepids[taskfn], tasknames, depends, mc) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 752 | |
| 753 | # Resolve inter-task dependencies |
| 754 | # |
| 755 | # e.g. do_sometask[depends] = "targetname:do_someothertask" |
| 756 | # (makes sure sometask runs after targetname's someothertask) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 757 | idepends = taskData[mc].taskentries[tid].idepends |
| 758 | for (depname, idependtask) in idepends: |
| 759 | if depname in taskData[mc].build_targets and taskData[mc].build_targets[depname] and not depname in taskData[mc].failed_deps: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 760 | # Won't be in build_targets if ASSUME_PROVIDED |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 761 | depdata = taskData[mc].build_targets[depname][0] |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 762 | if depdata is not None: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 763 | t = depdata + ":" + idependtask |
| 764 | depends.add(t) |
| 765 | if t not in taskData[mc].taskentries: |
| 766 | bb.msg.fatal("RunQueue", "Task %s in %s depends upon non-existent task %s in %s" % (taskname, fn, idependtask, depdata)) |
| 767 | irdepends = taskData[mc].taskentries[tid].irdepends |
| 768 | for (depname, idependtask) in irdepends: |
| 769 | if depname in taskData[mc].run_targets: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 770 | # Won't be in run_targets if ASSUME_PROVIDED |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 771 | if not taskData[mc].run_targets[depname]: |
| 772 | continue |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 773 | depdata = taskData[mc].run_targets[depname][0] |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 774 | if depdata is not None: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 775 | t = depdata + ":" + idependtask |
| 776 | depends.add(t) |
| 777 | if t not in taskData[mc].taskentries: |
| 778 | bb.msg.fatal("RunQueue", "Task %s in %s rdepends upon non-existent task %s in %s" % (taskname, fn, idependtask, depdata)) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 779 | |
| 780 | # Resolve recursive 'recrdeptask' dependencies (Part A) |
| 781 | # |
| 782 | # e.g. do_sometask[recrdeptask] = "do_someothertask" |
| 783 | # (makes sure sometask runs after someothertask of all DEPENDS, RDEPENDS and intertask dependencies, recursively) |
| 784 | # We cover the recursive part of the dependencies below |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 785 | if 'recrdeptask' in task_deps and taskname in task_deps['recrdeptask']: |
| 786 | tasknames = task_deps['recrdeptask'][taskname].split() |
| 787 | recursivetasks[tid] = tasknames |
| 788 | add_build_dependencies(taskData[mc].depids[taskfn], tasknames, depends, mc) |
| 789 | add_runtime_dependencies(taskData[mc].rdepids[taskfn], tasknames, depends, mc) |
| 790 | if taskname in tasknames: |
| 791 | recursivetasksselfref.add(tid) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 792 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 793 | if 'recideptask' in task_deps and taskname in task_deps['recideptask']: |
| 794 | recursiveitasks[tid] = [] |
| 795 | for t in task_deps['recideptask'][taskname].split(): |
| 796 | newdep = build_tid(mc, fn, t) |
| 797 | recursiveitasks[tid].append(newdep) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 798 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 799 | self.runtaskentries[tid].depends = depends |
Brad Bishop | 316dfdd | 2018-06-25 12:45:53 -0400 | [diff] [blame] | 800 | # Remove all self references |
| 801 | self.runtaskentries[tid].depends.discard(tid) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 802 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 803 | #self.dump_data() |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 804 | |
Brad Bishop | 316dfdd | 2018-06-25 12:45:53 -0400 | [diff] [blame] | 805 | self.init_progress_reporter.next_stage() |
| 806 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 807 | # Resolve recursive 'recrdeptask' dependencies (Part B) |
| 808 | # |
| 809 | # e.g. do_sometask[recrdeptask] = "do_someothertask" |
| 810 | # (makes sure sometask runs after someothertask of all DEPENDS, RDEPENDS and intertask dependencies, recursively) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 811 | # We need to do this separately since we need all of runtaskentries[*].depends to be complete before this is processed |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 812 | |
Brad Bishop | 316dfdd | 2018-06-25 12:45:53 -0400 | [diff] [blame] | 813 | # Generating/interating recursive lists of dependencies is painful and potentially slow |
| 814 | # Precompute recursive task dependencies here by: |
| 815 | # a) create a temp list of reverse dependencies (revdeps) |
| 816 | # b) walk up the ends of the chains (when a given task no longer has dependencies i.e. len(deps) == 0) |
| 817 | # c) combine the total list of dependencies in cumulativedeps |
| 818 | # d) optimise by pre-truncating 'task' off the items in cumulativedeps (keeps items in sets lower) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 819 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 820 | |
Brad Bishop | 316dfdd | 2018-06-25 12:45:53 -0400 | [diff] [blame] | 821 | revdeps = {} |
| 822 | deps = {} |
| 823 | cumulativedeps = {} |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 824 | for tid in self.runtaskentries: |
Brad Bishop | 316dfdd | 2018-06-25 12:45:53 -0400 | [diff] [blame] | 825 | deps[tid] = set(self.runtaskentries[tid].depends) |
| 826 | revdeps[tid] = set() |
| 827 | cumulativedeps[tid] = set() |
| 828 | # Generate a temp list of reverse dependencies |
| 829 | for tid in self.runtaskentries: |
| 830 | for dep in self.runtaskentries[tid].depends: |
| 831 | revdeps[dep].add(tid) |
| 832 | # Find the dependency chain endpoints |
| 833 | endpoints = set() |
| 834 | for tid in self.runtaskentries: |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 835 | if not deps[tid]: |
Brad Bishop | 316dfdd | 2018-06-25 12:45:53 -0400 | [diff] [blame] | 836 | endpoints.add(tid) |
| 837 | # Iterate the chains collating dependencies |
| 838 | while endpoints: |
| 839 | next = set() |
| 840 | for tid in endpoints: |
| 841 | for dep in revdeps[tid]: |
| 842 | cumulativedeps[dep].add(fn_from_tid(tid)) |
| 843 | cumulativedeps[dep].update(cumulativedeps[tid]) |
| 844 | if tid in deps[dep]: |
| 845 | deps[dep].remove(tid) |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 846 | if not deps[dep]: |
Brad Bishop | 316dfdd | 2018-06-25 12:45:53 -0400 | [diff] [blame] | 847 | next.add(dep) |
| 848 | endpoints = next |
| 849 | #for tid in deps: |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 850 | # if deps[tid]: |
Brad Bishop | 316dfdd | 2018-06-25 12:45:53 -0400 | [diff] [blame] | 851 | # bb.warn("Sanity test failure, dependencies left for %s (%s)" % (tid, deps[tid])) |
| 852 | |
| 853 | # Loop here since recrdeptasks can depend upon other recrdeptasks and we have to |
| 854 | # resolve these recursively until we aren't adding any further extra dependencies |
| 855 | extradeps = True |
| 856 | while extradeps: |
| 857 | extradeps = 0 |
| 858 | for tid in recursivetasks: |
| 859 | tasknames = recursivetasks[tid] |
| 860 | |
| 861 | totaldeps = set(self.runtaskentries[tid].depends) |
| 862 | if tid in recursiveitasks: |
| 863 | totaldeps.update(recursiveitasks[tid]) |
| 864 | for dep in recursiveitasks[tid]: |
| 865 | if dep not in self.runtaskentries: |
| 866 | continue |
| 867 | totaldeps.update(self.runtaskentries[dep].depends) |
| 868 | |
| 869 | deps = set() |
| 870 | for dep in totaldeps: |
| 871 | if dep in cumulativedeps: |
| 872 | deps.update(cumulativedeps[dep]) |
| 873 | |
| 874 | for t in deps: |
| 875 | for taskname in tasknames: |
| 876 | newtid = t + ":" + taskname |
| 877 | if newtid == tid: |
| 878 | continue |
| 879 | if newtid in self.runtaskentries and newtid not in self.runtaskentries[tid].depends: |
| 880 | extradeps += 1 |
| 881 | self.runtaskentries[tid].depends.add(newtid) |
| 882 | |
| 883 | # Handle recursive tasks which depend upon other recursive tasks |
| 884 | deps = set() |
| 885 | for dep in self.runtaskentries[tid].depends.intersection(recursivetasks): |
| 886 | deps.update(self.runtaskentries[dep].depends.difference(self.runtaskentries[tid].depends)) |
| 887 | for newtid in deps: |
| 888 | for taskname in tasknames: |
| 889 | if not newtid.endswith(":" + taskname): |
| 890 | continue |
| 891 | if newtid in self.runtaskentries: |
| 892 | extradeps += 1 |
| 893 | self.runtaskentries[tid].depends.add(newtid) |
| 894 | |
| 895 | bb.debug(1, "Added %s recursive dependencies in this loop" % extradeps) |
| 896 | |
| 897 | # Remove recrdeptask circular references so that do_a[recrdeptask] = "do_a do_b" can work |
| 898 | for tid in recursivetasksselfref: |
| 899 | self.runtaskentries[tid].depends.difference_update(recursivetasksselfref) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 900 | |
| 901 | self.init_progress_reporter.next_stage() |
| 902 | |
| 903 | #self.dump_data() |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 904 | |
| 905 | # Step B - Mark all active tasks |
| 906 | # |
| 907 | # Start with the tasks we were asked to run and mark all dependencies |
| 908 | # as active too. If the task is to be 'forced', clear its stamp. Once |
| 909 | # all active tasks are marked, prune the ones we don't need. |
| 910 | |
| 911 | logger.verbose("Marking Active Tasks") |
| 912 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 913 | def mark_active(tid, depth): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 914 | """ |
| 915 | Mark an item as active along with its depends |
| 916 | (calls itself recursively) |
| 917 | """ |
| 918 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 919 | if tid in runq_build: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 920 | return |
| 921 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 922 | runq_build[tid] = 1 |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 923 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 924 | depends = self.runtaskentries[tid].depends |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 925 | for depend in depends: |
| 926 | mark_active(depend, depth+1) |
| 927 | |
Brad Bishop | 79641f2 | 2019-09-10 07:20:22 -0400 | [diff] [blame] | 928 | def invalidate_task(tid, error_nostamp): |
| 929 | (mc, fn, taskname, taskfn) = split_tid_mcfn(tid) |
| 930 | taskdep = self.dataCaches[mc].task_deps[taskfn] |
| 931 | if fn + ":" + taskname not in taskData[mc].taskentries: |
| 932 | logger.warning("Task %s does not exist, invalidating this task will have no effect" % taskname) |
| 933 | if 'nostamp' in taskdep and taskname in taskdep['nostamp']: |
| 934 | if error_nostamp: |
| 935 | bb.fatal("Task %s is marked nostamp, cannot invalidate this task" % taskname) |
| 936 | else: |
| 937 | bb.debug(1, "Task %s is marked nostamp, cannot invalidate this task" % taskname) |
| 938 | else: |
| 939 | logger.verbose("Invalidate task %s, %s", taskname, fn) |
Andrew Geissler | 517393d | 2023-01-13 08:55:19 -0600 | [diff] [blame] | 940 | bb.parse.siggen.invalidate_task(taskname, taskfn) |
Brad Bishop | 79641f2 | 2019-09-10 07:20:22 -0400 | [diff] [blame] | 941 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 942 | self.target_tids = [] |
| 943 | for (mc, target, task, fn) in self.targets: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 944 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 945 | if target not in taskData[mc].build_targets or not taskData[mc].build_targets[target]: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 946 | continue |
| 947 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 948 | if target in taskData[mc].failed_deps: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 949 | continue |
| 950 | |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 951 | parents = False |
| 952 | if task.endswith('-'): |
| 953 | parents = True |
| 954 | task = task[:-1] |
| 955 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 956 | if fn in taskData[mc].failed_fns: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 957 | continue |
| 958 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 959 | # fn already has mc prefix |
| 960 | tid = fn + ":" + task |
| 961 | self.target_tids.append(tid) |
| 962 | if tid not in taskData[mc].taskentries: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 963 | import difflib |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 964 | tasks = [] |
| 965 | for x in taskData[mc].taskentries: |
| 966 | if x.startswith(fn + ":"): |
| 967 | tasks.append(taskname_from_tid(x)) |
| 968 | close_matches = difflib.get_close_matches(task, tasks, cutoff=0.7) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 969 | if close_matches: |
| 970 | extra = ". Close matches:\n %s" % "\n ".join(close_matches) |
| 971 | else: |
| 972 | extra = "" |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 973 | bb.msg.fatal("RunQueue", "Task %s does not exist for target %s (%s)%s" % (task, target, tid, extra)) |
| 974 | |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 975 | # For tasks called "XXXX-", ony run their dependencies |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 976 | if parents: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 977 | for i in self.runtaskentries[tid].depends: |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 978 | mark_active(i, 1) |
| 979 | else: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 980 | mark_active(tid, 1) |
| 981 | |
| 982 | self.init_progress_reporter.next_stage() |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 983 | |
| 984 | # Step C - Prune all inactive tasks |
| 985 | # |
| 986 | # Once all active tasks are marked, prune the ones we don't need. |
| 987 | |
Brad Bishop | 316dfdd | 2018-06-25 12:45:53 -0400 | [diff] [blame] | 988 | # Handle --runall |
| 989 | if self.cooker.configuration.runall: |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 990 | # re-run the mark_active and then drop unused tasks from new list |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 991 | reduced_tasklist = set(self.runtaskentries.keys()) |
| 992 | for tid in list(self.runtaskentries.keys()): |
| 993 | if tid not in runq_build: |
| 994 | reduced_tasklist.remove(tid) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 995 | runq_build = {} |
Brad Bishop | 316dfdd | 2018-06-25 12:45:53 -0400 | [diff] [blame] | 996 | |
| 997 | for task in self.cooker.configuration.runall: |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 998 | if not task.startswith("do_"): |
| 999 | task = "do_{0}".format(task) |
Brad Bishop | 316dfdd | 2018-06-25 12:45:53 -0400 | [diff] [blame] | 1000 | runall_tids = set() |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 1001 | for tid in reduced_tasklist: |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 1002 | wanttid = "{0}:{1}".format(fn_from_tid(tid), task) |
Brad Bishop | 316dfdd | 2018-06-25 12:45:53 -0400 | [diff] [blame] | 1003 | if wanttid in self.runtaskentries: |
| 1004 | runall_tids.add(wanttid) |
| 1005 | |
| 1006 | for tid in list(runall_tids): |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 1007 | mark_active(tid, 1) |
Brad Bishop | 79641f2 | 2019-09-10 07:20:22 -0400 | [diff] [blame] | 1008 | if self.cooker.configuration.force: |
| 1009 | invalidate_task(tid, False) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 1010 | |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 1011 | delcount = set() |
| 1012 | for tid in list(self.runtaskentries.keys()): |
| 1013 | if tid not in runq_build: |
| 1014 | delcount.add(tid) |
| 1015 | del self.runtaskentries[tid] |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 1016 | |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 1017 | if self.cooker.configuration.runall: |
| 1018 | if not self.runtaskentries: |
Brad Bishop | 316dfdd | 2018-06-25 12:45:53 -0400 | [diff] [blame] | 1019 | bb.msg.fatal("RunQueue", "Could not find any tasks with the tasknames %s to run within the recipes of the taskgraphs of the targets %s" % (str(self.cooker.configuration.runall), str(self.targets))) |
| 1020 | |
| 1021 | self.init_progress_reporter.next_stage() |
| 1022 | |
| 1023 | # Handle runonly |
| 1024 | if self.cooker.configuration.runonly: |
| 1025 | # re-run the mark_active and then drop unused tasks from new list |
| 1026 | runq_build = {} |
| 1027 | |
| 1028 | for task in self.cooker.configuration.runonly: |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 1029 | if not task.startswith("do_"): |
| 1030 | task = "do_{0}".format(task) |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 1031 | runonly_tids = [k for k in self.runtaskentries.keys() if taskname_from_tid(k) == task] |
Brad Bishop | 316dfdd | 2018-06-25 12:45:53 -0400 | [diff] [blame] | 1032 | |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 1033 | for tid in runonly_tids: |
| 1034 | mark_active(tid, 1) |
Brad Bishop | 79641f2 | 2019-09-10 07:20:22 -0400 | [diff] [blame] | 1035 | if self.cooker.configuration.force: |
| 1036 | invalidate_task(tid, False) |
Brad Bishop | 316dfdd | 2018-06-25 12:45:53 -0400 | [diff] [blame] | 1037 | |
| 1038 | for tid in list(self.runtaskentries.keys()): |
| 1039 | if tid not in runq_build: |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 1040 | delcount.add(tid) |
Brad Bishop | 316dfdd | 2018-06-25 12:45:53 -0400 | [diff] [blame] | 1041 | del self.runtaskentries[tid] |
| 1042 | |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 1043 | if not self.runtaskentries: |
Brad Bishop | 316dfdd | 2018-06-25 12:45:53 -0400 | [diff] [blame] | 1044 | bb.msg.fatal("RunQueue", "Could not find any tasks with the tasknames %s to run within the taskgraphs of the targets %s" % (str(self.cooker.configuration.runonly), str(self.targets))) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 1045 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1046 | # |
| 1047 | # Step D - Sanity checks and computation |
| 1048 | # |
| 1049 | |
| 1050 | # Check to make sure we still have tasks to run |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 1051 | if not self.runtaskentries: |
Andrew Geissler | 7e0e3c0 | 2022-02-25 20:34:39 +0000 | [diff] [blame] | 1052 | if not taskData[''].halt: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1053 | bb.msg.fatal("RunQueue", "All buildable tasks have been run but the build is incomplete (--continue mode). Errors for the tasks that failed will have been printed above.") |
| 1054 | else: |
| 1055 | bb.msg.fatal("RunQueue", "No active tasks and not in --continue mode?! Please report this bug.") |
| 1056 | |
Brad Bishop | 316dfdd | 2018-06-25 12:45:53 -0400 | [diff] [blame] | 1057 | logger.verbose("Pruned %s inactive tasks, %s left", len(delcount), len(self.runtaskentries)) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1058 | |
| 1059 | logger.verbose("Assign Weightings") |
| 1060 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1061 | self.init_progress_reporter.next_stage() |
| 1062 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1063 | # Generate a list of reverse dependencies to ease future calculations |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1064 | for tid in self.runtaskentries: |
| 1065 | for dep in self.runtaskentries[tid].depends: |
| 1066 | self.runtaskentries[dep].revdeps.add(tid) |
| 1067 | |
| 1068 | self.init_progress_reporter.next_stage() |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1069 | |
| 1070 | # Identify tasks at the end of dependency chains |
| 1071 | # Error on circular dependency loops (length two) |
| 1072 | endpoints = [] |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1073 | for tid in self.runtaskentries: |
| 1074 | revdeps = self.runtaskentries[tid].revdeps |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 1075 | if not revdeps: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1076 | endpoints.append(tid) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1077 | for dep in revdeps: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1078 | if dep in self.runtaskentries[tid].depends: |
| 1079 | bb.msg.fatal("RunQueue", "Task %s has circular dependency on %s" % (tid, dep)) |
| 1080 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1081 | |
| 1082 | logger.verbose("Compute totals (have %s endpoint(s))", len(endpoints)) |
| 1083 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1084 | self.init_progress_reporter.next_stage() |
| 1085 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1086 | # Calculate task weights |
| 1087 | # Check of higher length circular dependencies |
| 1088 | self.runq_weight = self.calculate_task_weights(endpoints) |
| 1089 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1090 | self.init_progress_reporter.next_stage() |
| 1091 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1092 | # Sanity Check - Check for multiple tasks building the same provider |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1093 | for mc in self.dataCaches: |
| 1094 | prov_list = {} |
| 1095 | seen_fn = [] |
| 1096 | for tid in self.runtaskentries: |
| 1097 | (tidmc, fn, taskname, taskfn) = split_tid_mcfn(tid) |
| 1098 | if taskfn in seen_fn: |
| 1099 | continue |
| 1100 | if mc != tidmc: |
| 1101 | continue |
| 1102 | seen_fn.append(taskfn) |
| 1103 | for prov in self.dataCaches[mc].fn_provides[taskfn]: |
| 1104 | if prov not in prov_list: |
| 1105 | prov_list[prov] = [taskfn] |
| 1106 | elif taskfn not in prov_list[prov]: |
| 1107 | prov_list[prov].append(taskfn) |
| 1108 | for prov in prov_list: |
| 1109 | if len(prov_list[prov]) < 2: |
| 1110 | continue |
Andrew Geissler | 7e0e3c0 | 2022-02-25 20:34:39 +0000 | [diff] [blame] | 1111 | if prov in self.multi_provider_allowed: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1112 | continue |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1113 | seen_pn = [] |
| 1114 | # If two versions of the same PN are being built its fatal, we don't support it. |
| 1115 | for fn in prov_list[prov]: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1116 | pn = self.dataCaches[mc].pkg_fn[fn] |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1117 | if pn not in seen_pn: |
| 1118 | seen_pn.append(pn) |
| 1119 | else: |
| 1120 | bb.fatal("Multiple versions of %s are due to be built (%s). Only one version of a given PN should be built in any given build. You likely need to set PREFERRED_VERSION_%s to select the correct version or don't depend on multiple versions." % (pn, " ".join(prov_list[prov]), pn)) |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 1121 | msgs = ["Multiple .bb files are due to be built which each provide %s:\n %s" % (prov, "\n ".join(prov_list[prov]))] |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 1122 | # |
| 1123 | # Construct a list of things which uniquely depend on each provider |
| 1124 | # since this may help the user figure out which dependency is triggering this warning |
| 1125 | # |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 1126 | msgs.append("\nA list of tasks depending on these providers is shown and may help explain where the dependency comes from.") |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 1127 | deplist = {} |
| 1128 | commondeps = None |
| 1129 | for provfn in prov_list[prov]: |
| 1130 | deps = set() |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1131 | for tid in self.runtaskentries: |
| 1132 | fn = fn_from_tid(tid) |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 1133 | if fn != provfn: |
| 1134 | continue |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1135 | for dep in self.runtaskentries[tid].revdeps: |
| 1136 | fn = fn_from_tid(dep) |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 1137 | if fn == provfn: |
| 1138 | continue |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1139 | deps.add(dep) |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 1140 | if not commondeps: |
| 1141 | commondeps = set(deps) |
| 1142 | else: |
| 1143 | commondeps &= deps |
| 1144 | deplist[provfn] = deps |
| 1145 | for provfn in deplist: |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 1146 | msgs.append("\n%s has unique dependees:\n %s" % (provfn, "\n ".join(deplist[provfn] - commondeps))) |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 1147 | # |
| 1148 | # Construct a list of provides and runtime providers for each recipe |
| 1149 | # (rprovides has to cover RPROVIDES, PACKAGES, PACKAGES_DYNAMIC) |
| 1150 | # |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 1151 | msgs.append("\nIt could be that one recipe provides something the other doesn't and should. The following provider and runtime provider differences may be helpful.") |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 1152 | provide_results = {} |
| 1153 | rprovide_results = {} |
| 1154 | commonprovs = None |
| 1155 | commonrprovs = None |
| 1156 | for provfn in prov_list[prov]: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1157 | provides = set(self.dataCaches[mc].fn_provides[provfn]) |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 1158 | rprovides = set() |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1159 | for rprovide in self.dataCaches[mc].rproviders: |
| 1160 | if provfn in self.dataCaches[mc].rproviders[rprovide]: |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 1161 | rprovides.add(rprovide) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1162 | for package in self.dataCaches[mc].packages: |
| 1163 | if provfn in self.dataCaches[mc].packages[package]: |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 1164 | rprovides.add(package) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1165 | for package in self.dataCaches[mc].packages_dynamic: |
| 1166 | if provfn in self.dataCaches[mc].packages_dynamic[package]: |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 1167 | rprovides.add(package) |
| 1168 | if not commonprovs: |
| 1169 | commonprovs = set(provides) |
| 1170 | else: |
| 1171 | commonprovs &= provides |
| 1172 | provide_results[provfn] = provides |
| 1173 | if not commonrprovs: |
| 1174 | commonrprovs = set(rprovides) |
| 1175 | else: |
| 1176 | commonrprovs &= rprovides |
| 1177 | rprovide_results[provfn] = rprovides |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 1178 | #msgs.append("\nCommon provides:\n %s" % ("\n ".join(commonprovs))) |
| 1179 | #msgs.append("\nCommon rprovides:\n %s" % ("\n ".join(commonrprovs))) |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 1180 | for provfn in prov_list[prov]: |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 1181 | msgs.append("\n%s has unique provides:\n %s" % (provfn, "\n ".join(provide_results[provfn] - commonprovs))) |
| 1182 | msgs.append("\n%s has unique rprovides:\n %s" % (provfn, "\n ".join(rprovide_results[provfn] - commonrprovs))) |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 1183 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1184 | if self.warn_multi_bb: |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 1185 | logger.verbnote("".join(msgs)) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1186 | else: |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 1187 | logger.error("".join(msgs)) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1188 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1189 | self.init_progress_reporter.next_stage() |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1190 | self.init_progress_reporter.next_stage() |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1191 | |
| 1192 | # Iterate over the task list looking for tasks with a 'setscene' function |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 1193 | self.runq_setscene_tids = set() |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1194 | if not self.cooker.configuration.nosetscene: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1195 | for tid in self.runtaskentries: |
| 1196 | (mc, fn, taskname, _) = split_tid_mcfn(tid) |
Brad Bishop | 37a0e4d | 2017-12-04 01:01:44 -0500 | [diff] [blame] | 1197 | setscenetid = tid + "_setscene" |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1198 | if setscenetid not in taskData[mc].taskentries: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1199 | continue |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 1200 | self.runq_setscene_tids.add(tid) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1201 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1202 | self.init_progress_reporter.next_stage() |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1203 | |
| 1204 | # Invalidate task if force mode active |
| 1205 | if self.cooker.configuration.force: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1206 | for tid in self.target_tids: |
| 1207 | invalidate_task(tid, False) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1208 | |
| 1209 | # Invalidate task if invalidate mode active |
| 1210 | if self.cooker.configuration.invalidate_stamp: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1211 | for tid in self.target_tids: |
| 1212 | fn = fn_from_tid(tid) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1213 | for st in self.cooker.configuration.invalidate_stamp.split(','): |
| 1214 | if not st.startswith("do_"): |
| 1215 | st = "do_%s" % st |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1216 | invalidate_task(fn + ":" + st, True) |
| 1217 | |
| 1218 | self.init_progress_reporter.next_stage() |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1219 | |
Patrick Williams | f1e5d69 | 2016-03-30 15:21:19 -0500 | [diff] [blame] | 1220 | # Create and print to the logs a virtual/xxxx -> PN (fn) table |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1221 | for mc in taskData: |
| 1222 | virtmap = taskData[mc].get_providermap(prefix="virtual/") |
| 1223 | virtpnmap = {} |
| 1224 | for v in virtmap: |
| 1225 | virtpnmap[v] = self.dataCaches[mc].pkg_fn[virtmap[v]] |
| 1226 | bb.debug(2, "%s resolved to: %s (%s)" % (v, virtpnmap[v], virtmap[v])) |
| 1227 | if hasattr(bb.parse.siggen, "tasks_resolved"): |
| 1228 | bb.parse.siggen.tasks_resolved(virtmap, virtpnmap, self.dataCaches[mc]) |
| 1229 | |
| 1230 | self.init_progress_reporter.next_stage() |
Patrick Williams | f1e5d69 | 2016-03-30 15:21:19 -0500 | [diff] [blame] | 1231 | |
Brad Bishop | 00e122a | 2019-10-05 11:10:57 -0400 | [diff] [blame] | 1232 | bb.parse.siggen.set_setscene_tasks(self.runq_setscene_tids) |
| 1233 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1234 | # Iterate over the task list and call into the siggen code |
| 1235 | dealtwith = set() |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1236 | todeal = set(self.runtaskentries) |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 1237 | while todeal: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1238 | for tid in todeal.copy(): |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 1239 | if not (self.runtaskentries[tid].depends - dealtwith): |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1240 | dealtwith.add(tid) |
| 1241 | todeal.remove(tid) |
Brad Bishop | 1932369 | 2019-04-05 15:28:33 -0400 | [diff] [blame] | 1242 | self.prepare_task_hash(tid) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1243 | |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 1244 | bb.parse.siggen.writeout_file_checksum_cache() |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1245 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1246 | #self.dump_data() |
| 1247 | return len(self.runtaskentries) |
| 1248 | |
Brad Bishop | 1932369 | 2019-04-05 15:28:33 -0400 | [diff] [blame] | 1249 | def prepare_task_hash(self, tid): |
Andrew Geissler | 517393d | 2023-01-13 08:55:19 -0600 | [diff] [blame] | 1250 | bb.parse.siggen.prep_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches) |
| 1251 | self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches) |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 1252 | self.runtaskentries[tid].unihash = bb.parse.siggen.get_unihash(tid) |
Brad Bishop | 1932369 | 2019-04-05 15:28:33 -0400 | [diff] [blame] | 1253 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1254 | def dump_data(self): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1255 | """ |
| 1256 | Dump some debug information on the internal data structures |
| 1257 | """ |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 1258 | logger.debug3("run_tasks:") |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1259 | for tid in self.runtaskentries: |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 1260 | logger.debug3(" %s: %s Deps %s RevDeps %s", tid, |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1261 | self.runtaskentries[tid].weight, |
| 1262 | self.runtaskentries[tid].depends, |
| 1263 | self.runtaskentries[tid].revdeps) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1264 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1265 | class RunQueueWorker(): |
| 1266 | def __init__(self, process, pipe): |
| 1267 | self.process = process |
| 1268 | self.pipe = pipe |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1269 | |
| 1270 | class RunQueue: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1271 | def __init__(self, cooker, cfgData, dataCaches, taskData, targets): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1272 | |
| 1273 | self.cooker = cooker |
| 1274 | self.cfgData = cfgData |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1275 | self.rqdata = RunQueueData(self, cooker, cfgData, dataCaches, taskData, targets) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1276 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 1277 | self.hashvalidate = cfgData.getVar("BB_HASHCHECK_FUNCTION") or None |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 1278 | self.depvalidate = cfgData.getVar("BB_SETSCENE_DEPVALID") or None |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1279 | |
| 1280 | self.state = runQueuePrepare |
| 1281 | |
| 1282 | # For disk space monitor |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 1283 | # Invoked at regular time intervals via the bitbake heartbeat event |
| 1284 | # while the build is running. We generate a unique name for the handler |
| 1285 | # here, just in case that there ever is more than one RunQueue instance, |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 1286 | # start the handler when reaching runQueueSceneInit, and stop it when |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 1287 | # done with the build. |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1288 | self.dm = monitordisk.diskMonitor(cfgData) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 1289 | self.dm_event_handler_name = '_bb_diskmonitor_' + str(id(self)) |
| 1290 | self.dm_event_handler_registered = False |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1291 | self.rqexe = None |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1292 | self.worker = {} |
| 1293 | self.fakeworker = {} |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1294 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1295 | def _start_worker(self, mc, fakeroot = False, rqexec = None): |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 1296 | logger.debug("Starting bitbake-worker") |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1297 | magic = "decafbad" |
| 1298 | if self.cooker.configuration.profile: |
| 1299 | magic = "decafbadbad" |
Andrew Geissler | 95ac1b8 | 2021-03-31 14:34:31 -0500 | [diff] [blame] | 1300 | fakerootlogs = None |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1301 | if fakeroot: |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 1302 | magic = magic + "beef" |
Brad Bishop | 37a0e4d | 2017-12-04 01:01:44 -0500 | [diff] [blame] | 1303 | mcdata = self.cooker.databuilder.mcdata[mc] |
Brad Bishop | 1932369 | 2019-04-05 15:28:33 -0400 | [diff] [blame] | 1304 | fakerootcmd = shlex.split(mcdata.getVar("FAKEROOTCMD")) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 1305 | fakerootenv = (mcdata.getVar("FAKEROOTBASEENV") or "").split() |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1306 | env = os.environ.copy() |
| 1307 | for key, value in (var.split('=') for var in fakerootenv): |
| 1308 | env[key] = value |
Brad Bishop | 1932369 | 2019-04-05 15:28:33 -0400 | [diff] [blame] | 1309 | worker = subprocess.Popen(fakerootcmd + ["bitbake-worker", magic], stdout=subprocess.PIPE, stdin=subprocess.PIPE, env=env) |
Andrew Geissler | 95ac1b8 | 2021-03-31 14:34:31 -0500 | [diff] [blame] | 1310 | fakerootlogs = self.rqdata.dataCaches[mc].fakerootlogs |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1311 | else: |
| 1312 | worker = subprocess.Popen(["bitbake-worker", magic], stdout=subprocess.PIPE, stdin=subprocess.PIPE) |
| 1313 | bb.utils.nonblockingfd(worker.stdout) |
Andrew Geissler | 95ac1b8 | 2021-03-31 14:34:31 -0500 | [diff] [blame] | 1314 | workerpipe = runQueuePipe(worker.stdout, None, self.cfgData, self, rqexec, fakerootlogs=fakerootlogs) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1315 | |
| 1316 | workerdata = { |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1317 | "sigdata" : bb.parse.siggen.get_taskdata(), |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 1318 | "logdefaultlevel" : bb.msg.loggerDefaultLogLevel, |
Andrew Geissler | c9f7865 | 2020-09-18 14:11:35 -0500 | [diff] [blame] | 1319 | "build_verbose_shell" : self.cooker.configuration.build_verbose_shell, |
| 1320 | "build_verbose_stdout" : self.cooker.configuration.build_verbose_stdout, |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1321 | "logdefaultdomain" : bb.msg.loggerDefaultDomains, |
| 1322 | "prhost" : self.cooker.prhost, |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 1323 | "buildname" : self.cfgData.getVar("BUILDNAME"), |
| 1324 | "date" : self.cfgData.getVar("DATE"), |
| 1325 | "time" : self.cfgData.getVar("TIME"), |
Brad Bishop | a34c030 | 2019-09-23 22:34:48 -0400 | [diff] [blame] | 1326 | "hashservaddr" : self.cooker.hashservaddr, |
Andrew Geissler | 9b4d8b0 | 2021-02-19 12:26:16 -0600 | [diff] [blame] | 1327 | "umask" : self.cfgData.getVar("BB_DEFAULT_UMASK"), |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1328 | } |
| 1329 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1330 | worker.stdin.write(b"<cookerconfig>" + pickle.dumps(self.cooker.configuration) + b"</cookerconfig>") |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 1331 | worker.stdin.write(b"<extraconfigdata>" + pickle.dumps(self.cooker.extraconfigdata) + b"</extraconfigdata>") |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1332 | worker.stdin.write(b"<workerdata>" + pickle.dumps(workerdata) + b"</workerdata>") |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1333 | worker.stdin.flush() |
| 1334 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1335 | return RunQueueWorker(worker, workerpipe) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1336 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1337 | def _teardown_worker(self, worker): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1338 | if not worker: |
| 1339 | return |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 1340 | logger.debug("Teardown for bitbake-worker") |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1341 | try: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1342 | worker.process.stdin.write(b"<quit></quit>") |
| 1343 | worker.process.stdin.flush() |
| 1344 | worker.process.stdin.close() |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1345 | except IOError: |
| 1346 | pass |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1347 | while worker.process.returncode is None: |
| 1348 | worker.pipe.read() |
| 1349 | worker.process.poll() |
| 1350 | while worker.pipe.read(): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1351 | continue |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1352 | worker.pipe.close() |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1353 | |
| 1354 | def start_worker(self): |
| 1355 | if self.worker: |
| 1356 | self.teardown_workers() |
| 1357 | self.teardown = False |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1358 | for mc in self.rqdata.dataCaches: |
| 1359 | self.worker[mc] = self._start_worker(mc) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1360 | |
Brad Bishop | 37a0e4d | 2017-12-04 01:01:44 -0500 | [diff] [blame] | 1361 | def start_fakeworker(self, rqexec, mc): |
| 1362 | if not mc in self.fakeworker: |
| 1363 | self.fakeworker[mc] = self._start_worker(mc, True, rqexec) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1364 | |
| 1365 | def teardown_workers(self): |
| 1366 | self.teardown = True |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1367 | for mc in self.worker: |
| 1368 | self._teardown_worker(self.worker[mc]) |
| 1369 | self.worker = {} |
| 1370 | for mc in self.fakeworker: |
| 1371 | self._teardown_worker(self.fakeworker[mc]) |
| 1372 | self.fakeworker = {} |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1373 | |
| 1374 | def read_workers(self): |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1375 | for mc in self.worker: |
| 1376 | self.worker[mc].pipe.read() |
| 1377 | for mc in self.fakeworker: |
| 1378 | self.fakeworker[mc].pipe.read() |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1379 | |
| 1380 | def active_fds(self): |
| 1381 | fds = [] |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1382 | for mc in self.worker: |
| 1383 | fds.append(self.worker[mc].pipe.input) |
| 1384 | for mc in self.fakeworker: |
| 1385 | fds.append(self.fakeworker[mc].pipe.input) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1386 | return fds |
| 1387 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1388 | def check_stamp_task(self, tid, taskname = None, recurse = False, cache = None): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1389 | def get_timestamp(f): |
| 1390 | try: |
| 1391 | if not os.access(f, os.F_OK): |
| 1392 | return None |
| 1393 | return os.stat(f)[stat.ST_MTIME] |
| 1394 | except: |
| 1395 | return None |
| 1396 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1397 | (mc, fn, tn, taskfn) = split_tid_mcfn(tid) |
| 1398 | if taskname is None: |
| 1399 | taskname = tn |
| 1400 | |
Andrew Geissler | 517393d | 2023-01-13 08:55:19 -0600 | [diff] [blame] | 1401 | stampfile = bb.parse.siggen.stampfile_mcfn(taskname, taskfn) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1402 | |
| 1403 | # If the stamp is missing, it's not current |
| 1404 | if not os.access(stampfile, os.F_OK): |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 1405 | logger.debug2("Stampfile %s not available", stampfile) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1406 | return False |
| 1407 | # If it's a 'nostamp' task, it's not current |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1408 | taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn] |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1409 | if 'nostamp' in taskdep and taskname in taskdep['nostamp']: |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 1410 | logger.debug2("%s.%s is nostamp\n", fn, taskname) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1411 | return False |
| 1412 | |
Andrew Geissler | 517393d | 2023-01-13 08:55:19 -0600 | [diff] [blame] | 1413 | if taskname.endswith("_setscene"): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1414 | return True |
| 1415 | |
| 1416 | if cache is None: |
| 1417 | cache = {} |
| 1418 | |
| 1419 | iscurrent = True |
| 1420 | t1 = get_timestamp(stampfile) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1421 | for dep in self.rqdata.runtaskentries[tid].depends: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1422 | if iscurrent: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1423 | (mc2, fn2, taskname2, taskfn2) = split_tid_mcfn(dep) |
Andrew Geissler | 517393d | 2023-01-13 08:55:19 -0600 | [diff] [blame] | 1424 | stampfile2 = bb.parse.siggen.stampfile_mcfn(taskname2, taskfn2) |
| 1425 | stampfile3 = bb.parse.siggen.stampfile_mcfn(taskname2 + "_setscene", taskfn2) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1426 | t2 = get_timestamp(stampfile2) |
| 1427 | t3 = get_timestamp(stampfile3) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1428 | if t3 and not t2: |
| 1429 | continue |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1430 | if t3 and t3 > t2: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1431 | continue |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 1432 | if fn == fn2: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1433 | if not t2: |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 1434 | logger.debug2('Stampfile %s does not exist', stampfile2) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1435 | iscurrent = False |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1436 | break |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1437 | if t1 < t2: |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 1438 | logger.debug2('Stampfile %s < %s', stampfile, stampfile2) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1439 | iscurrent = False |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1440 | break |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1441 | if recurse and iscurrent: |
| 1442 | if dep in cache: |
| 1443 | iscurrent = cache[dep] |
| 1444 | if not iscurrent: |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 1445 | logger.debug2('Stampfile for dependency %s:%s invalid (cached)' % (fn2, taskname2)) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1446 | else: |
| 1447 | iscurrent = self.check_stamp_task(dep, recurse=True, cache=cache) |
| 1448 | cache[dep] = iscurrent |
| 1449 | if recurse: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1450 | cache[tid] = iscurrent |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1451 | return iscurrent |
| 1452 | |
Brad Bishop | 1d80a2e | 2019-11-15 16:35:03 -0500 | [diff] [blame] | 1453 | def validate_hashes(self, tocheck, data, currentcount=0, siginfo=False, summary=True): |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 1454 | valid = set() |
| 1455 | if self.hashvalidate: |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 1456 | sq_data = {} |
| 1457 | sq_data['hash'] = {} |
| 1458 | sq_data['hashfn'] = {} |
| 1459 | sq_data['unihash'] = {} |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 1460 | for tid in tocheck: |
| 1461 | (mc, fn, taskname, taskfn) = split_tid_mcfn(tid) |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 1462 | sq_data['hash'][tid] = self.rqdata.runtaskentries[tid].hash |
| 1463 | sq_data['hashfn'][tid] = self.rqdata.dataCaches[mc].hashfn[taskfn] |
| 1464 | sq_data['unihash'][tid] = self.rqdata.runtaskentries[tid].unihash |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 1465 | |
Brad Bishop | 1d80a2e | 2019-11-15 16:35:03 -0500 | [diff] [blame] | 1466 | valid = self.validate_hash(sq_data, data, siginfo, currentcount, summary) |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 1467 | |
| 1468 | return valid |
| 1469 | |
Brad Bishop | 1d80a2e | 2019-11-15 16:35:03 -0500 | [diff] [blame] | 1470 | def validate_hash(self, sq_data, d, siginfo, currentcount, summary): |
| 1471 | locs = {"sq_data" : sq_data, "d" : d, "siginfo" : siginfo, "currentcount" : currentcount, "summary" : summary} |
Brad Bishop | 1932369 | 2019-04-05 15:28:33 -0400 | [diff] [blame] | 1472 | |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 1473 | # Metadata has **kwargs so args can be added, sq_data can also gain new fields |
Brad Bishop | 1d80a2e | 2019-11-15 16:35:03 -0500 | [diff] [blame] | 1474 | call = self.hashvalidate + "(sq_data, d, siginfo=siginfo, currentcount=currentcount, summary=summary)" |
Brad Bishop | 1932369 | 2019-04-05 15:28:33 -0400 | [diff] [blame] | 1475 | |
Brad Bishop | 1932369 | 2019-04-05 15:28:33 -0400 | [diff] [blame] | 1476 | return bb.utils.better_eval(call, locs) |
| 1477 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1478 | def _execute_runqueue(self): |
| 1479 | """ |
| 1480 | Run the tasks in a queue prepared by rqdata.prepare() |
| 1481 | Upon failure, optionally try to recover the build using any alternate providers |
Andrew Geissler | 7e0e3c0 | 2022-02-25 20:34:39 +0000 | [diff] [blame] | 1482 | (if the halt on failure configuration option isn't set) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1483 | """ |
| 1484 | |
| 1485 | retval = True |
| 1486 | |
| 1487 | if self.state is runQueuePrepare: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1488 | # NOTE: if you add, remove or significantly refactor the stages of this |
| 1489 | # process then you should recalculate the weightings here. This is quite |
| 1490 | # easy to do - just change the next line temporarily to pass debug=True as |
| 1491 | # the last parameter and you'll get a printout of the weightings as well |
| 1492 | # as a map to the lines where next_stage() was called. Of course this isn't |
| 1493 | # critical, but it helps to keep the progress reporting accurate. |
| 1494 | self.rqdata.init_progress_reporter = bb.progress.MultiStageProcessProgressReporter(self.cooker.data, |
| 1495 | "Initialising tasks", |
| 1496 | [43, 967, 4, 3, 1, 5, 3, 7, 13, 1, 2, 1, 1, 246, 35, 1, 38, 1, 35, 2, 338, 204, 142, 3, 3, 37, 244]) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1497 | if self.rqdata.prepare() == 0: |
| 1498 | self.state = runQueueComplete |
| 1499 | else: |
| 1500 | self.state = runQueueSceneInit |
Brad Bishop | 00e122a | 2019-10-05 11:10:57 -0400 | [diff] [blame] | 1501 | bb.parse.siggen.save_unitaskhashes() |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1502 | |
| 1503 | if self.state is runQueueSceneInit: |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 1504 | self.rqdata.init_progress_reporter.next_stage() |
| 1505 | |
| 1506 | # we are ready to run, emit dependency info to any UI or class which |
| 1507 | # needs it |
| 1508 | depgraph = self.cooker.buildDependTree(self, self.rqdata.taskData) |
| 1509 | self.rqdata.init_progress_reporter.next_stage() |
| 1510 | bb.event.fire(bb.event.DepTreeGenerated(depgraph), self.cooker.data) |
| 1511 | |
Brad Bishop | e2d5b61 | 2018-11-23 10:55:50 +1300 | [diff] [blame] | 1512 | if not self.dm_event_handler_registered: |
| 1513 | res = bb.event.register(self.dm_event_handler_name, |
Andrew Geissler | 517393d | 2023-01-13 08:55:19 -0600 | [diff] [blame] | 1514 | lambda x, y: self.dm.check(self) if self.state in [runQueueRunning, runQueueCleanUp] else False, |
Andrew Geissler | 9b4d8b0 | 2021-02-19 12:26:16 -0600 | [diff] [blame] | 1515 | ('bb.event.HeartbeatEvent',), data=self.cfgData) |
Brad Bishop | e2d5b61 | 2018-11-23 10:55:50 +1300 | [diff] [blame] | 1516 | self.dm_event_handler_registered = True |
| 1517 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1518 | dump = self.cooker.configuration.dump_signatures |
| 1519 | if dump: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1520 | self.rqdata.init_progress_reporter.finish() |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1521 | if 'printdiff' in dump: |
| 1522 | invalidtasks = self.print_diffscenetasks() |
| 1523 | self.dump_signatures(dump) |
| 1524 | if 'printdiff' in dump: |
| 1525 | self.write_diffscenetasks(invalidtasks) |
| 1526 | self.state = runQueueComplete |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1527 | |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 1528 | if self.state is runQueueSceneInit: |
| 1529 | self.rqdata.init_progress_reporter.next_stage() |
| 1530 | self.start_worker() |
| 1531 | self.rqdata.init_progress_reporter.next_stage() |
| 1532 | self.rqexe = RunQueueExecute(self) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1533 | |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 1534 | # If we don't have any setscene functions, skip execution |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 1535 | if not self.rqdata.runq_setscene_tids: |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 1536 | logger.info('No setscene tasks') |
| 1537 | for tid in self.rqdata.runtaskentries: |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 1538 | if not self.rqdata.runtaskentries[tid].depends: |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 1539 | self.rqexe.setbuildable(tid) |
| 1540 | self.rqexe.tasks_notcovered.add(tid) |
| 1541 | self.rqexe.sqdone = True |
| 1542 | logger.info('Executing Tasks') |
| 1543 | self.state = runQueueRunning |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1544 | |
| 1545 | if self.state is runQueueRunning: |
| 1546 | retval = self.rqexe.execute() |
| 1547 | |
| 1548 | if self.state is runQueueCleanUp: |
| 1549 | retval = self.rqexe.finish() |
| 1550 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 1551 | build_done = self.state is runQueueComplete or self.state is runQueueFailed |
| 1552 | |
| 1553 | if build_done and self.dm_event_handler_registered: |
Andrew Geissler | 9b4d8b0 | 2021-02-19 12:26:16 -0600 | [diff] [blame] | 1554 | bb.event.remove(self.dm_event_handler_name, None, data=self.cfgData) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 1555 | self.dm_event_handler_registered = False |
| 1556 | |
| 1557 | if build_done and self.rqexe: |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 1558 | bb.parse.siggen.save_unitaskhashes() |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1559 | self.teardown_workers() |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 1560 | if self.rqexe: |
| 1561 | if self.rqexe.stats.failed: |
| 1562 | logger.info("Tasks Summary: Attempted %d tasks of which %d didn't need to be rerun and %d failed.", self.rqexe.stats.completed + self.rqexe.stats.failed, self.rqexe.stats.skipped, self.rqexe.stats.failed) |
| 1563 | else: |
| 1564 | # Let's avoid the word "failed" if nothing actually did |
| 1565 | logger.info("Tasks Summary: Attempted %d tasks of which %d didn't need to be rerun and all succeeded.", self.rqexe.stats.completed, self.rqexe.stats.skipped) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1566 | |
| 1567 | if self.state is runQueueFailed: |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 1568 | raise bb.runqueue.TaskFailure(self.rqexe.failed_tids) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1569 | |
| 1570 | if self.state is runQueueComplete: |
| 1571 | # All done |
| 1572 | return False |
| 1573 | |
| 1574 | # Loop |
| 1575 | return retval |
| 1576 | |
| 1577 | def execute_runqueue(self): |
| 1578 | # Catch unexpected exceptions and ensure we exit when an error occurs, not loop. |
| 1579 | try: |
| 1580 | return self._execute_runqueue() |
| 1581 | except bb.runqueue.TaskFailure: |
| 1582 | raise |
| 1583 | except SystemExit: |
| 1584 | raise |
| 1585 | except bb.BBHandledException: |
| 1586 | try: |
| 1587 | self.teardown_workers() |
| 1588 | except: |
| 1589 | pass |
| 1590 | self.state = runQueueComplete |
| 1591 | raise |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1592 | except Exception as err: |
| 1593 | logger.exception("An uncaught exception occurred in runqueue") |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1594 | try: |
| 1595 | self.teardown_workers() |
| 1596 | except: |
| 1597 | pass |
| 1598 | self.state = runQueueComplete |
| 1599 | raise |
| 1600 | |
| 1601 | def finish_runqueue(self, now = False): |
| 1602 | if not self.rqexe: |
| 1603 | self.state = runQueueComplete |
| 1604 | return |
| 1605 | |
| 1606 | if now: |
| 1607 | self.rqexe.finish_now() |
| 1608 | else: |
| 1609 | self.rqexe.finish() |
| 1610 | |
Andrew Geissler | 517393d | 2023-01-13 08:55:19 -0600 | [diff] [blame] | 1611 | def _rq_dump_sigtid(self, tids): |
| 1612 | for tid in tids: |
| 1613 | (mc, fn, taskname, taskfn) = split_tid_mcfn(tid) |
| 1614 | dataCaches = self.rqdata.dataCaches |
| 1615 | bb.parse.siggen.dump_sigtask(taskfn, taskname, dataCaches[mc].stamp[taskfn], True) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 1616 | |
| 1617 | def dump_signatures(self, options): |
Andrew Geissler | 517393d | 2023-01-13 08:55:19 -0600 | [diff] [blame] | 1618 | if bb.cooker.CookerFeatures.RECIPE_SIGGEN_INFO not in self.cooker.featureset: |
| 1619 | bb.fatal("The dump signatures functionality needs the RECIPE_SIGGEN_INFO feature enabled") |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 1620 | |
Andrew Geissler | 517393d | 2023-01-13 08:55:19 -0600 | [diff] [blame] | 1621 | bb.note("Writing task signature files") |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 1622 | |
| 1623 | max_process = int(self.cfgData.getVar("BB_NUMBER_PARSE_THREADS") or os.cpu_count() or 1) |
Andrew Geissler | 517393d | 2023-01-13 08:55:19 -0600 | [diff] [blame] | 1624 | def chunkify(l, n): |
| 1625 | return [l[i::n] for i in range(n)] |
| 1626 | tids = chunkify(list(self.rqdata.runtaskentries), max_process) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 1627 | # We cannot use the real multiprocessing.Pool easily due to some local data |
| 1628 | # that can't be pickled. This is a cheap multi-process solution. |
| 1629 | launched = [] |
Andrew Geissler | 517393d | 2023-01-13 08:55:19 -0600 | [diff] [blame] | 1630 | while tids: |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 1631 | if len(launched) < max_process: |
Andrew Geissler | 517393d | 2023-01-13 08:55:19 -0600 | [diff] [blame] | 1632 | p = Process(target=self._rq_dump_sigtid, args=(tids.pop(), )) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 1633 | p.start() |
| 1634 | launched.append(p) |
| 1635 | for q in launched: |
| 1636 | # The finished processes are joined when calling is_alive() |
| 1637 | if not q.is_alive(): |
| 1638 | launched.remove(q) |
| 1639 | for p in launched: |
| 1640 | p.join() |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1641 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1642 | bb.parse.siggen.dump_sigs(self.rqdata.dataCaches, options) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1643 | |
| 1644 | return |
| 1645 | |
| 1646 | def print_diffscenetasks(self): |
| 1647 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1648 | noexec = [] |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 1649 | tocheck = set() |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1650 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1651 | for tid in self.rqdata.runtaskentries: |
| 1652 | (mc, fn, taskname, taskfn) = split_tid_mcfn(tid) |
| 1653 | taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn] |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1654 | |
| 1655 | if 'noexec' in taskdep and taskname in taskdep['noexec']: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1656 | noexec.append(tid) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1657 | continue |
| 1658 | |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 1659 | tocheck.add(tid) |
Brad Bishop | 1932369 | 2019-04-05 15:28:33 -0400 | [diff] [blame] | 1660 | |
Brad Bishop | 1d80a2e | 2019-11-15 16:35:03 -0500 | [diff] [blame] | 1661 | valid_new = self.validate_hashes(tocheck, self.cooker.data, 0, True, summary=False) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1662 | |
| 1663 | # Tasks which are both setscene and noexec never care about dependencies |
| 1664 | # We therefore find tasks which are setscene and noexec and mark their |
| 1665 | # unique dependencies as valid. |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1666 | for tid in noexec: |
| 1667 | if tid not in self.rqdata.runq_setscene_tids: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1668 | continue |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1669 | for dep in self.rqdata.runtaskentries[tid].depends: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1670 | hasnoexecparents = True |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1671 | for dep2 in self.rqdata.runtaskentries[dep].revdeps: |
| 1672 | if dep2 in self.rqdata.runq_setscene_tids and dep2 in noexec: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1673 | continue |
| 1674 | hasnoexecparents = False |
| 1675 | break |
| 1676 | if hasnoexecparents: |
| 1677 | valid_new.add(dep) |
| 1678 | |
| 1679 | invalidtasks = set() |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1680 | for tid in self.rqdata.runtaskentries: |
| 1681 | if tid not in valid_new and tid not in noexec: |
| 1682 | invalidtasks.add(tid) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1683 | |
| 1684 | found = set() |
| 1685 | processed = set() |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1686 | for tid in invalidtasks: |
| 1687 | toprocess = set([tid]) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1688 | while toprocess: |
| 1689 | next = set() |
| 1690 | for t in toprocess: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1691 | for dep in self.rqdata.runtaskentries[t].depends: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1692 | if dep in invalidtasks: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1693 | found.add(tid) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1694 | if dep not in processed: |
| 1695 | processed.add(dep) |
| 1696 | next.add(dep) |
| 1697 | toprocess = next |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1698 | if tid in found: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1699 | toprocess = set() |
| 1700 | |
| 1701 | tasklist = [] |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1702 | for tid in invalidtasks.difference(found): |
| 1703 | tasklist.append(tid) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1704 | |
| 1705 | if tasklist: |
| 1706 | bb.plain("The differences between the current build and any cached tasks start at the following tasks:\n" + "\n".join(tasklist)) |
| 1707 | |
| 1708 | return invalidtasks.difference(found) |
| 1709 | |
| 1710 | def write_diffscenetasks(self, invalidtasks): |
| 1711 | |
| 1712 | # Define recursion callback |
| 1713 | def recursecb(key, hash1, hash2): |
| 1714 | hashes = [hash1, hash2] |
| 1715 | hashfiles = bb.siggen.find_siginfo(key, None, hashes, self.cfgData) |
| 1716 | |
| 1717 | recout = [] |
| 1718 | if len(hashfiles) == 2: |
| 1719 | out2 = bb.siggen.compare_sigfiles(hashfiles[hash1], hashfiles[hash2], recursecb) |
Brad Bishop | c342db3 | 2019-05-15 21:57:59 -0400 | [diff] [blame] | 1720 | recout.extend(list(' ' + l for l in out2)) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1721 | else: |
| 1722 | recout.append("Unable to find matching sigdata for %s with hashes %s or %s" % (key, hash1, hash2)) |
| 1723 | |
| 1724 | return recout |
| 1725 | |
| 1726 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1727 | for tid in invalidtasks: |
Brad Bishop | 37a0e4d | 2017-12-04 01:01:44 -0500 | [diff] [blame] | 1728 | (mc, fn, taskname, taskfn) = split_tid_mcfn(tid) |
| 1729 | pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn] |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1730 | h = self.rqdata.runtaskentries[tid].hash |
Patrick Williams | 03907ee | 2022-05-01 06:28:52 -0500 | [diff] [blame] | 1731 | matches = bb.siggen.find_siginfo(pn, taskname, [], self.cooker.databuilder.mcdata[mc]) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1732 | match = None |
| 1733 | for m in matches: |
| 1734 | if h in m: |
| 1735 | match = m |
| 1736 | if match is None: |
| 1737 | bb.fatal("Can't find a task we're supposed to have written out? (hash: %s)?" % h) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1738 | matches = {k : v for k, v in iter(matches.items()) if h not in k} |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1739 | if matches: |
| 1740 | latestmatch = sorted(matches.keys(), key=lambda f: matches[f])[-1] |
Brad Bishop | 1932369 | 2019-04-05 15:28:33 -0400 | [diff] [blame] | 1741 | prevh = __find_sha256__.search(latestmatch).group(0) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1742 | output = bb.siggen.compare_sigfiles(latestmatch, match, recursecb) |
| 1743 | bb.plain("\nTask %s:%s couldn't be used from the cache because:\n We need hash %s, closest matching task was %s\n " % (pn, taskname, h, prevh) + '\n '.join(output)) |
| 1744 | |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 1745 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1746 | class RunQueueExecute: |
| 1747 | |
| 1748 | def __init__(self, rq): |
| 1749 | self.rq = rq |
| 1750 | self.cooker = rq.cooker |
| 1751 | self.cfgData = rq.cfgData |
| 1752 | self.rqdata = rq.rqdata |
| 1753 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 1754 | self.number_tasks = int(self.cfgData.getVar("BB_NUMBER_THREADS") or 1) |
| 1755 | self.scheduler = self.cfgData.getVar("BB_SCHEDULER") or "speed" |
Patrick Williams | db4c27e | 2022-08-05 08:10:29 -0500 | [diff] [blame] | 1756 | self.max_cpu_pressure = self.cfgData.getVar("BB_PRESSURE_MAX_CPU") |
| 1757 | self.max_io_pressure = self.cfgData.getVar("BB_PRESSURE_MAX_IO") |
Patrick Williams | 92b42cb | 2022-09-03 06:53:57 -0500 | [diff] [blame] | 1758 | self.max_memory_pressure = self.cfgData.getVar("BB_PRESSURE_MAX_MEMORY") |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1759 | |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 1760 | self.sq_buildable = set() |
| 1761 | self.sq_running = set() |
| 1762 | self.sq_live = set() |
| 1763 | |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 1764 | self.updated_taskhash_queue = [] |
| 1765 | self.pending_migrations = set() |
| 1766 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1767 | self.runq_buildable = set() |
| 1768 | self.runq_running = set() |
| 1769 | self.runq_complete = set() |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 1770 | self.runq_tasksrun = set() |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1771 | |
| 1772 | self.build_stamps = {} |
| 1773 | self.build_stamps2 = [] |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1774 | self.failed_tids = [] |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 1775 | self.sq_deferred = {} |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1776 | |
| 1777 | self.stampcache = {} |
| 1778 | |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 1779 | self.holdoff_tasks = set() |
Brad Bishop | c68388fc | 2019-08-26 01:33:31 -0400 | [diff] [blame] | 1780 | self.holdoff_need_update = True |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 1781 | self.sqdone = False |
| 1782 | |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 1783 | self.stats = RunQueueStats(len(self.rqdata.runtaskentries), len(self.rqdata.runq_setscene_tids)) |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 1784 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1785 | for mc in rq.worker: |
| 1786 | rq.worker[mc].pipe.setrunqueueexec(self) |
| 1787 | for mc in rq.fakeworker: |
| 1788 | rq.fakeworker[mc].pipe.setrunqueueexec(self) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1789 | |
| 1790 | if self.number_tasks <= 0: |
| 1791 | bb.fatal("Invalid BB_NUMBER_THREADS %s" % self.number_tasks) |
| 1792 | |
Patrick Williams | db4c27e | 2022-08-05 08:10:29 -0500 | [diff] [blame] | 1793 | lower_limit = 1.0 |
| 1794 | upper_limit = 1000000.0 |
| 1795 | if self.max_cpu_pressure: |
| 1796 | self.max_cpu_pressure = float(self.max_cpu_pressure) |
| 1797 | if self.max_cpu_pressure < lower_limit: |
| 1798 | bb.fatal("Invalid BB_PRESSURE_MAX_CPU %s, minimum value is %s." % (self.max_cpu_pressure, lower_limit)) |
| 1799 | if self.max_cpu_pressure > upper_limit: |
| 1800 | bb.warn("Your build will be largely unregulated since BB_PRESSURE_MAX_CPU is set to %s. It is very unlikely that such high pressure will be experienced." % (self.max_cpu_pressure)) |
| 1801 | |
| 1802 | if self.max_io_pressure: |
| 1803 | self.max_io_pressure = float(self.max_io_pressure) |
| 1804 | if self.max_io_pressure < lower_limit: |
| 1805 | bb.fatal("Invalid BB_PRESSURE_MAX_IO %s, minimum value is %s." % (self.max_io_pressure, lower_limit)) |
| 1806 | if self.max_io_pressure > upper_limit: |
| 1807 | bb.warn("Your build will be largely unregulated since BB_PRESSURE_MAX_IO is set to %s. It is very unlikely that such high pressure will be experienced." % (self.max_io_pressure)) |
| 1808 | |
Patrick Williams | 92b42cb | 2022-09-03 06:53:57 -0500 | [diff] [blame] | 1809 | if self.max_memory_pressure: |
| 1810 | self.max_memory_pressure = float(self.max_memory_pressure) |
| 1811 | if self.max_memory_pressure < lower_limit: |
| 1812 | bb.fatal("Invalid BB_PRESSURE_MAX_MEMORY %s, minimum value is %s." % (self.max_memory_pressure, lower_limit)) |
| 1813 | if self.max_memory_pressure > upper_limit: |
| 1814 | bb.warn("Your build will be largely unregulated since BB_PRESSURE_MAX_MEMORY is set to %s. It is very unlikely that such high pressure will be experienced." % (self.max_io_pressure)) |
| 1815 | |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 1816 | # List of setscene tasks which we've covered |
| 1817 | self.scenequeue_covered = set() |
| 1818 | # List of tasks which are covered (including setscene ones) |
| 1819 | self.tasks_covered = set() |
| 1820 | self.tasks_scenequeue_done = set() |
| 1821 | self.scenequeue_notcovered = set() |
| 1822 | self.tasks_notcovered = set() |
| 1823 | self.scenequeue_notneeded = set() |
| 1824 | |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 1825 | # We can't skip specified target tasks which aren't setscene tasks |
| 1826 | self.cantskip = set(self.rqdata.target_tids) |
| 1827 | self.cantskip.difference_update(self.rqdata.runq_setscene_tids) |
| 1828 | self.cantskip.intersection_update(self.rqdata.runtaskentries) |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 1829 | |
| 1830 | schedulers = self.get_schedulers() |
| 1831 | for scheduler in schedulers: |
| 1832 | if self.scheduler == scheduler.name: |
| 1833 | self.sched = scheduler(self, self.rqdata) |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 1834 | logger.debug("Using runqueue scheduler '%s'", scheduler.name) |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 1835 | break |
| 1836 | else: |
| 1837 | bb.fatal("Invalid scheduler '%s'. Available schedulers: %s" % |
| 1838 | (self.scheduler, ", ".join(obj.name for obj in schedulers))) |
| 1839 | |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 1840 | #if self.rqdata.runq_setscene_tids: |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 1841 | self.sqdata = SQData() |
| 1842 | build_scenequeue_data(self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self) |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 1843 | |
Andrew Geissler | 95ac1b8 | 2021-03-31 14:34:31 -0500 | [diff] [blame] | 1844 | def runqueue_process_waitpid(self, task, status, fakerootlog=None): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1845 | |
| 1846 | # self.build_stamps[pid] may not exist when use shared work directory. |
| 1847 | if task in self.build_stamps: |
| 1848 | self.build_stamps2.remove(self.build_stamps[task]) |
| 1849 | del self.build_stamps[task] |
| 1850 | |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 1851 | if task in self.sq_live: |
| 1852 | if status != 0: |
| 1853 | self.sq_task_fail(task, status) |
| 1854 | else: |
| 1855 | self.sq_task_complete(task) |
| 1856 | self.sq_live.remove(task) |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 1857 | self.stats.updateActiveSetscene(len(self.sq_live)) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1858 | else: |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 1859 | if status != 0: |
Andrew Geissler | 95ac1b8 | 2021-03-31 14:34:31 -0500 | [diff] [blame] | 1860 | self.task_fail(task, status, fakerootlog=fakerootlog) |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 1861 | else: |
| 1862 | self.task_complete(task) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1863 | return True |
| 1864 | |
| 1865 | def finish_now(self): |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1866 | for mc in self.rq.worker: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1867 | try: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1868 | self.rq.worker[mc].process.stdin.write(b"<finishnow></finishnow>") |
| 1869 | self.rq.worker[mc].process.stdin.flush() |
| 1870 | except IOError: |
| 1871 | # worker must have died? |
| 1872 | pass |
| 1873 | for mc in self.rq.fakeworker: |
| 1874 | try: |
| 1875 | self.rq.fakeworker[mc].process.stdin.write(b"<finishnow></finishnow>") |
| 1876 | self.rq.fakeworker[mc].process.stdin.flush() |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1877 | except IOError: |
| 1878 | # worker must have died? |
| 1879 | pass |
| 1880 | |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 1881 | if self.failed_tids: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1882 | self.rq.state = runQueueFailed |
| 1883 | return |
| 1884 | |
| 1885 | self.rq.state = runQueueComplete |
| 1886 | return |
| 1887 | |
| 1888 | def finish(self): |
| 1889 | self.rq.state = runQueueCleanUp |
| 1890 | |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 1891 | active = self.stats.active + len(self.sq_live) |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 1892 | if active > 0: |
| 1893 | bb.event.fire(runQueueExitWait(active), self.cfgData) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1894 | self.rq.read_workers() |
| 1895 | return self.rq.active_fds() |
| 1896 | |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 1897 | if self.failed_tids: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1898 | self.rq.state = runQueueFailed |
| 1899 | return True |
| 1900 | |
| 1901 | self.rq.state = runQueueComplete |
| 1902 | return True |
| 1903 | |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 1904 | # Used by setscene only |
| 1905 | def check_dependencies(self, task, taskdeps): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1906 | if not self.rq.depvalidate: |
| 1907 | return False |
| 1908 | |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 1909 | # Must not edit parent data |
| 1910 | taskdeps = set(taskdeps) |
| 1911 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1912 | taskdata = {} |
| 1913 | taskdeps.add(task) |
| 1914 | for dep in taskdeps: |
Brad Bishop | 37a0e4d | 2017-12-04 01:01:44 -0500 | [diff] [blame] | 1915 | (mc, fn, taskname, taskfn) = split_tid_mcfn(dep) |
| 1916 | pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn] |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1917 | taskdata[dep] = [pn, taskname, fn] |
| 1918 | call = self.rq.depvalidate + "(task, taskdata, notneeded, d)" |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 1919 | locs = { "task" : task, "taskdata" : taskdata, "notneeded" : self.scenequeue_notneeded, "d" : self.cooker.data } |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1920 | valid = bb.utils.better_eval(call, locs) |
| 1921 | return valid |
| 1922 | |
Brad Bishop | 1a4b7ee | 2018-12-16 17:11:34 -0800 | [diff] [blame] | 1923 | def can_start_task(self): |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 1924 | active = self.stats.active + len(self.sq_live) |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 1925 | can_start = active < self.number_tasks |
Brad Bishop | 1a4b7ee | 2018-12-16 17:11:34 -0800 | [diff] [blame] | 1926 | return can_start |
| 1927 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1928 | def get_schedulers(self): |
| 1929 | schedulers = set(obj for obj in globals().values() |
| 1930 | if type(obj) is type and |
| 1931 | issubclass(obj, RunQueueScheduler)) |
| 1932 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 1933 | user_schedulers = self.cfgData.getVar("BB_SCHEDULERS") |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1934 | if user_schedulers: |
| 1935 | for sched in user_schedulers.split(): |
| 1936 | if not "." in sched: |
| 1937 | bb.note("Ignoring scheduler '%s' from BB_SCHEDULERS: not an import" % sched) |
| 1938 | continue |
| 1939 | |
| 1940 | modname, name = sched.rsplit(".", 1) |
| 1941 | try: |
| 1942 | module = __import__(modname, fromlist=(name,)) |
| 1943 | except ImportError as exc: |
| 1944 | logger.critical("Unable to import scheduler '%s' from '%s': %s" % (name, modname, exc)) |
| 1945 | raise SystemExit(1) |
| 1946 | else: |
| 1947 | schedulers.add(getattr(module, name)) |
| 1948 | return schedulers |
| 1949 | |
| 1950 | def setbuildable(self, task): |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1951 | self.runq_buildable.add(task) |
Brad Bishop | 316dfdd | 2018-06-25 12:45:53 -0400 | [diff] [blame] | 1952 | self.sched.newbuildable(task) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1953 | |
| 1954 | def task_completeoutright(self, task): |
| 1955 | """ |
| 1956 | Mark a task as completed |
| 1957 | Look at the reverse dependencies and mark any task with |
| 1958 | completed dependencies as buildable |
| 1959 | """ |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1960 | self.runq_complete.add(task) |
| 1961 | for revdep in self.rqdata.runtaskentries[task].revdeps: |
| 1962 | if revdep in self.runq_running: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1963 | continue |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1964 | if revdep in self.runq_buildable: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1965 | continue |
Brad Bishop | 1a4b7ee | 2018-12-16 17:11:34 -0800 | [diff] [blame] | 1966 | alldeps = True |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1967 | for dep in self.rqdata.runtaskentries[revdep].depends: |
| 1968 | if dep not in self.runq_complete: |
Brad Bishop | 1a4b7ee | 2018-12-16 17:11:34 -0800 | [diff] [blame] | 1969 | alldeps = False |
| 1970 | break |
| 1971 | if alldeps: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1972 | self.setbuildable(revdep) |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 1973 | logger.debug("Marking task %s as buildable", revdep) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1974 | |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 1975 | for t in self.sq_deferred.copy(): |
| 1976 | if self.sq_deferred[t] == task: |
| 1977 | logger.debug2("Deferred task %s now buildable" % t) |
| 1978 | del self.sq_deferred[t] |
| 1979 | update_scenequeue_data([t], self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self, summary=False) |
| 1980 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1981 | def task_complete(self, task): |
| 1982 | self.stats.taskCompleted() |
| 1983 | bb.event.fire(runQueueTaskCompleted(task, self.stats, self.rq), self.cfgData) |
| 1984 | self.task_completeoutright(task) |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 1985 | self.runq_tasksrun.add(task) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1986 | |
Andrew Geissler | 95ac1b8 | 2021-03-31 14:34:31 -0500 | [diff] [blame] | 1987 | def task_fail(self, task, exitcode, fakerootlog=None): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1988 | """ |
| 1989 | Called when a task has failed |
| 1990 | Updates the state engine with the failure |
| 1991 | """ |
| 1992 | self.stats.taskFailed() |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1993 | self.failed_tids.append(task) |
Andrew Geissler | 95ac1b8 | 2021-03-31 14:34:31 -0500 | [diff] [blame] | 1994 | |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 1995 | fakeroot_log = [] |
Andrew Geissler | 95ac1b8 | 2021-03-31 14:34:31 -0500 | [diff] [blame] | 1996 | if fakerootlog and os.path.exists(fakerootlog): |
| 1997 | with open(fakerootlog) as fakeroot_log_file: |
| 1998 | fakeroot_failed = False |
| 1999 | for line in reversed(fakeroot_log_file.readlines()): |
| 2000 | for fakeroot_error in ['mismatch', 'error', 'fatal']: |
| 2001 | if fakeroot_error in line.lower(): |
| 2002 | fakeroot_failed = True |
| 2003 | if 'doing new pid setup and server start' in line: |
| 2004 | break |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 2005 | fakeroot_log.append(line) |
Andrew Geissler | 95ac1b8 | 2021-03-31 14:34:31 -0500 | [diff] [blame] | 2006 | |
| 2007 | if not fakeroot_failed: |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 2008 | fakeroot_log = [] |
Andrew Geissler | 95ac1b8 | 2021-03-31 14:34:31 -0500 | [diff] [blame] | 2009 | |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 2010 | bb.event.fire(runQueueTaskFailed(task, self.stats, exitcode, self.rq, fakeroot_log=("".join(fakeroot_log) or None)), self.cfgData) |
Andrew Geissler | 95ac1b8 | 2021-03-31 14:34:31 -0500 | [diff] [blame] | 2011 | |
Andrew Geissler | 7e0e3c0 | 2022-02-25 20:34:39 +0000 | [diff] [blame] | 2012 | if self.rqdata.taskData[''].halt: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2013 | self.rq.state = runQueueCleanUp |
| 2014 | |
| 2015 | def task_skip(self, task, reason): |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 2016 | self.runq_running.add(task) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2017 | self.setbuildable(task) |
| 2018 | bb.event.fire(runQueueTaskSkipped(task, self.stats, self.rq, reason), self.cfgData) |
| 2019 | self.task_completeoutright(task) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2020 | self.stats.taskSkipped() |
Brad Bishop | 1a4b7ee | 2018-12-16 17:11:34 -0800 | [diff] [blame] | 2021 | self.stats.taskCompleted() |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2022 | |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 2023 | def summarise_scenequeue_errors(self): |
| 2024 | err = False |
| 2025 | if not self.sqdone: |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 2026 | logger.debug('We could skip tasks %s', "\n".join(sorted(self.scenequeue_covered))) |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 2027 | completeevent = sceneQueueComplete(self.stats, self.rq) |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 2028 | bb.event.fire(completeevent, self.cfgData) |
| 2029 | if self.sq_deferred: |
| 2030 | logger.error("Scenequeue had deferred entries: %s" % pprint.pformat(self.sq_deferred)) |
| 2031 | err = True |
| 2032 | if self.updated_taskhash_queue: |
| 2033 | logger.error("Scenequeue had unprocessed changed taskhash entries: %s" % pprint.pformat(self.updated_taskhash_queue)) |
| 2034 | err = True |
| 2035 | if self.holdoff_tasks: |
| 2036 | logger.error("Scenequeue had holdoff tasks: %s" % pprint.pformat(self.holdoff_tasks)) |
| 2037 | err = True |
| 2038 | |
Andrew Geissler | 95ac1b8 | 2021-03-31 14:34:31 -0500 | [diff] [blame] | 2039 | for tid in self.scenequeue_covered.intersection(self.scenequeue_notcovered): |
| 2040 | # No task should end up in both covered and uncovered, that is a bug. |
| 2041 | logger.error("Setscene task %s in both covered and notcovered." % tid) |
| 2042 | |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 2043 | for tid in self.rqdata.runq_setscene_tids: |
| 2044 | if tid not in self.scenequeue_covered and tid not in self.scenequeue_notcovered: |
| 2045 | err = True |
| 2046 | logger.error("Setscene Task %s was never marked as covered or not covered" % tid) |
| 2047 | if tid not in self.sq_buildable: |
| 2048 | err = True |
| 2049 | logger.error("Setscene Task %s was never marked as buildable" % tid) |
| 2050 | if tid not in self.sq_running: |
| 2051 | err = True |
| 2052 | logger.error("Setscene Task %s was never marked as running" % tid) |
| 2053 | |
| 2054 | for x in self.rqdata.runtaskentries: |
| 2055 | if x not in self.tasks_covered and x not in self.tasks_notcovered: |
| 2056 | logger.error("Task %s was never moved from the setscene queue" % x) |
| 2057 | err = True |
| 2058 | if x not in self.tasks_scenequeue_done: |
| 2059 | logger.error("Task %s was never processed by the setscene code" % x) |
| 2060 | err = True |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 2061 | if not self.rqdata.runtaskentries[x].depends and x not in self.runq_buildable: |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 2062 | logger.error("Task %s was never marked as buildable by the setscene code" % x) |
| 2063 | err = True |
| 2064 | return err |
| 2065 | |
| 2066 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2067 | def execute(self): |
| 2068 | """ |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2069 | Run the tasks in a queue prepared by prepare_runqueue |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2070 | """ |
| 2071 | |
| 2072 | self.rq.read_workers() |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 2073 | if self.updated_taskhash_queue or self.pending_migrations: |
| 2074 | self.process_possible_migrations() |
| 2075 | |
| 2076 | if not hasattr(self, "sorted_setscene_tids"): |
| 2077 | # Don't want to sort this set every execution |
| 2078 | self.sorted_setscene_tids = sorted(self.rqdata.runq_setscene_tids) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2079 | |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2080 | task = None |
| 2081 | if not self.sqdone and self.can_start_task(): |
| 2082 | # Find the next setscene to run |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 2083 | for nexttask in self.sorted_setscene_tids: |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2084 | if nexttask in self.sq_buildable and nexttask not in self.sq_running and self.sqdata.stamps[nexttask] not in self.build_stamps.values(): |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 2085 | if nexttask not in self.sqdata.unskippable and self.sqdata.sq_revdeps[nexttask] and self.sqdata.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and self.check_dependencies(nexttask, self.sqdata.sq_revdeps[nexttask]): |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2086 | if nexttask not in self.rqdata.target_tids: |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 2087 | logger.debug2("Skipping setscene for task %s" % nexttask) |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2088 | self.sq_task_skip(nexttask) |
| 2089 | self.scenequeue_notneeded.add(nexttask) |
| 2090 | if nexttask in self.sq_deferred: |
| 2091 | del self.sq_deferred[nexttask] |
| 2092 | return True |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 2093 | # If covered tasks are running, need to wait for them to complete |
| 2094 | for t in self.sqdata.sq_covered_tasks[nexttask]: |
| 2095 | if t in self.runq_running and t not in self.runq_complete: |
| 2096 | continue |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2097 | if nexttask in self.sq_deferred: |
| 2098 | if self.sq_deferred[nexttask] not in self.runq_complete: |
| 2099 | continue |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 2100 | logger.debug("Task %s no longer deferred" % nexttask) |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2101 | del self.sq_deferred[nexttask] |
Brad Bishop | 1d80a2e | 2019-11-15 16:35:03 -0500 | [diff] [blame] | 2102 | valid = self.rq.validate_hashes(set([nexttask]), self.cooker.data, 0, False, summary=False) |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2103 | if not valid: |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 2104 | logger.debug("%s didn't become valid, skipping setscene" % nexttask) |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2105 | self.sq_task_failoutright(nexttask) |
| 2106 | return True |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2107 | if nexttask in self.sqdata.outrightfail: |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 2108 | logger.debug2('No package found, so skipping setscene task %s', nexttask) |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2109 | self.sq_task_failoutright(nexttask) |
| 2110 | return True |
| 2111 | if nexttask in self.sqdata.unskippable: |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 2112 | logger.debug2("Setscene task %s is unskippable" % nexttask) |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2113 | task = nexttask |
| 2114 | break |
| 2115 | if task is not None: |
| 2116 | (mc, fn, taskname, taskfn) = split_tid_mcfn(task) |
| 2117 | taskname = taskname + "_setscene" |
| 2118 | if self.rq.check_stamp_task(task, taskname_from_tid(task), recurse = True, cache=self.stampcache): |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 2119 | logger.debug2('Stamp for underlying task %s is current, so skipping setscene variant', task) |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2120 | self.sq_task_failoutright(task) |
| 2121 | return True |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2122 | |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2123 | if self.cooker.configuration.force: |
| 2124 | if task in self.rqdata.target_tids: |
| 2125 | self.sq_task_failoutright(task) |
| 2126 | return True |
| 2127 | |
| 2128 | if self.rq.check_stamp_task(task, taskname, cache=self.stampcache): |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 2129 | logger.debug2('Setscene stamp current task %s, so skip it and its dependencies', task) |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2130 | self.sq_task_skip(task) |
| 2131 | return True |
| 2132 | |
| 2133 | if self.cooker.configuration.skipsetscene: |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 2134 | logger.debug2('No setscene tasks should be executed. Skipping %s', task) |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2135 | self.sq_task_failoutright(task) |
| 2136 | return True |
| 2137 | |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 2138 | startevent = sceneQueueTaskStarted(task, self.stats, self.rq) |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2139 | bb.event.fire(startevent, self.cfgData) |
| 2140 | |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2141 | taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn] |
Andrew Geissler | 517393d | 2023-01-13 08:55:19 -0600 | [diff] [blame] | 2142 | runtask = { |
| 2143 | 'fn' : taskfn, |
| 2144 | 'task' : task, |
| 2145 | 'taskname' : taskname, |
| 2146 | 'taskhash' : self.rqdata.get_task_hash(task), |
| 2147 | 'unihash' : self.rqdata.get_task_unihash(task), |
| 2148 | 'quieterrors' : True, |
| 2149 | 'appends' : self.cooker.collections[mc].get_file_appends(taskfn), |
| 2150 | 'taskdepdata' : self.sq_build_taskdepdata(task), |
| 2151 | 'dry_run' : False, |
| 2152 | 'taskdep': taskdep, |
| 2153 | 'fakerootenv' : self.rqdata.dataCaches[mc].fakerootenv[taskfn], |
| 2154 | 'fakerootdirs' : self.rqdata.dataCaches[mc].fakerootdirs[taskfn], |
| 2155 | 'fakerootnoenv' : self.rqdata.dataCaches[mc].fakerootnoenv[taskfn] |
| 2156 | } |
| 2157 | |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2158 | if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run: |
| 2159 | if not mc in self.rq.fakeworker: |
| 2160 | self.rq.start_fakeworker(self, mc) |
Andrew Geissler | 517393d | 2023-01-13 08:55:19 -0600 | [diff] [blame] | 2161 | self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps(runtask) + b"</runtask>") |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2162 | self.rq.fakeworker[mc].process.stdin.flush() |
| 2163 | else: |
Andrew Geissler | 517393d | 2023-01-13 08:55:19 -0600 | [diff] [blame] | 2164 | self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps(runtask) + b"</runtask>") |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2165 | self.rq.worker[mc].process.stdin.flush() |
| 2166 | |
Andrew Geissler | 517393d | 2023-01-13 08:55:19 -0600 | [diff] [blame] | 2167 | self.build_stamps[task] = bb.parse.siggen.stampfile_mcfn(taskname, taskfn, extrainfo=False) |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2168 | self.build_stamps2.append(self.build_stamps[task]) |
| 2169 | self.sq_running.add(task) |
| 2170 | self.sq_live.add(task) |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 2171 | self.stats.updateActiveSetscene(len(self.sq_live)) |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2172 | if self.can_start_task(): |
| 2173 | return True |
| 2174 | |
Brad Bishop | c68388fc | 2019-08-26 01:33:31 -0400 | [diff] [blame] | 2175 | self.update_holdofftasks() |
| 2176 | |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 2177 | if not self.sq_live and not self.sqdone and not self.sq_deferred and not self.updated_taskhash_queue and not self.holdoff_tasks: |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 2178 | hashequiv_logger.verbose("Setscene tasks completed") |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2179 | |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 2180 | err = self.summarise_scenequeue_errors() |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2181 | if err: |
| 2182 | self.rq.state = runQueueFailed |
| 2183 | return True |
| 2184 | |
| 2185 | if self.cooker.configuration.setsceneonly: |
| 2186 | self.rq.state = runQueueComplete |
| 2187 | return True |
| 2188 | self.sqdone = True |
| 2189 | |
| 2190 | if self.stats.total == 0: |
| 2191 | # nothing to do |
| 2192 | self.rq.state = runQueueComplete |
| 2193 | return True |
| 2194 | |
| 2195 | if self.cooker.configuration.setsceneonly: |
| 2196 | task = None |
| 2197 | else: |
| 2198 | task = self.sched.next() |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2199 | if task is not None: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 2200 | (mc, fn, taskname, taskfn) = split_tid_mcfn(task) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2201 | |
Andrew Geissler | 7e0e3c0 | 2022-02-25 20:34:39 +0000 | [diff] [blame] | 2202 | if self.rqdata.setscene_ignore_tasks is not None: |
| 2203 | if self.check_setscene_ignore_tasks(task): |
| 2204 | self.task_fail(task, "setscene ignore_tasks") |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2205 | return True |
| 2206 | |
| 2207 | if task in self.tasks_covered: |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 2208 | logger.debug2("Setscene covered task %s", task) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2209 | self.task_skip(task, "covered") |
| 2210 | return True |
| 2211 | |
| 2212 | if self.rq.check_stamp_task(task, taskname, cache=self.stampcache): |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 2213 | logger.debug2("Stamp current task %s", task) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 2214 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2215 | self.task_skip(task, "existing") |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 2216 | self.runq_tasksrun.add(task) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2217 | return True |
| 2218 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 2219 | taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn] |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2220 | if 'noexec' in taskdep and taskname in taskdep['noexec']: |
| 2221 | startevent = runQueueTaskStarted(task, self.stats, self.rq, |
| 2222 | noexec=True) |
| 2223 | bb.event.fire(startevent, self.cfgData) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 2224 | self.runq_running.add(task) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2225 | self.stats.taskActive() |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 2226 | if not (self.cooker.configuration.dry_run or self.rqdata.setscene_enforce): |
Andrew Geissler | 517393d | 2023-01-13 08:55:19 -0600 | [diff] [blame] | 2227 | bb.build.make_stamp_mcfn(taskname, taskfn) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2228 | self.task_complete(task) |
| 2229 | return True |
| 2230 | else: |
| 2231 | startevent = runQueueTaskStarted(task, self.stats, self.rq) |
| 2232 | bb.event.fire(startevent, self.cfgData) |
| 2233 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 2234 | taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn] |
Andrew Geissler | 517393d | 2023-01-13 08:55:19 -0600 | [diff] [blame] | 2235 | runtask = { |
| 2236 | 'fn' : taskfn, |
| 2237 | 'task' : task, |
| 2238 | 'taskname' : taskname, |
| 2239 | 'taskhash' : self.rqdata.get_task_hash(task), |
| 2240 | 'unihash' : self.rqdata.get_task_unihash(task), |
| 2241 | 'quieterrors' : False, |
| 2242 | 'appends' : self.cooker.collections[mc].get_file_appends(taskfn), |
| 2243 | 'taskdepdata' : self.build_taskdepdata(task), |
| 2244 | 'dry_run' : self.rqdata.setscene_enforce, |
| 2245 | 'taskdep': taskdep, |
| 2246 | 'fakerootenv' : self.rqdata.dataCaches[mc].fakerootenv[taskfn], |
| 2247 | 'fakerootdirs' : self.rqdata.dataCaches[mc].fakerootdirs[taskfn], |
| 2248 | 'fakerootnoenv' : self.rqdata.dataCaches[mc].fakerootnoenv[taskfn] |
| 2249 | } |
| 2250 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 2251 | if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not (self.cooker.configuration.dry_run or self.rqdata.setscene_enforce): |
Brad Bishop | 37a0e4d | 2017-12-04 01:01:44 -0500 | [diff] [blame] | 2252 | if not mc in self.rq.fakeworker: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2253 | try: |
Brad Bishop | 37a0e4d | 2017-12-04 01:01:44 -0500 | [diff] [blame] | 2254 | self.rq.start_fakeworker(self, mc) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2255 | except OSError as exc: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 2256 | logger.critical("Failed to spawn fakeroot worker to run %s: %s" % (task, str(exc))) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2257 | self.rq.state = runQueueFailed |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 2258 | self.stats.taskFailed() |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2259 | return True |
Andrew Geissler | 517393d | 2023-01-13 08:55:19 -0600 | [diff] [blame] | 2260 | self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps(runtask) + b"</runtask>") |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 2261 | self.rq.fakeworker[mc].process.stdin.flush() |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2262 | else: |
Andrew Geissler | 517393d | 2023-01-13 08:55:19 -0600 | [diff] [blame] | 2263 | self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps(runtask) + b"</runtask>") |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 2264 | self.rq.worker[mc].process.stdin.flush() |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2265 | |
Andrew Geissler | 517393d | 2023-01-13 08:55:19 -0600 | [diff] [blame] | 2266 | self.build_stamps[task] = bb.parse.siggen.stampfile_mcfn(taskname, taskfn, extrainfo=False) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 2267 | self.build_stamps2.append(self.build_stamps[task]) |
| 2268 | self.runq_running.add(task) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2269 | self.stats.taskActive() |
Brad Bishop | 1a4b7ee | 2018-12-16 17:11:34 -0800 | [diff] [blame] | 2270 | if self.can_start_task(): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2271 | return True |
| 2272 | |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 2273 | if self.stats.active > 0 or self.sq_live: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2274 | self.rq.read_workers() |
| 2275 | return self.rq.active_fds() |
| 2276 | |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2277 | # No more tasks can be run. If we have deferred setscene tasks we should run them. |
| 2278 | if self.sq_deferred: |
Patrick Williams | 92b42cb | 2022-09-03 06:53:57 -0500 | [diff] [blame] | 2279 | deferred_tid = list(self.sq_deferred.keys())[0] |
| 2280 | blocking_tid = self.sq_deferred.pop(deferred_tid) |
Patrick Williams | 2390b1b | 2022-11-03 13:47:49 -0500 | [diff] [blame] | 2281 | logger.warning("Runqueue deadlocked on deferred tasks, forcing task %s blocked by %s" % (deferred_tid, blocking_tid)) |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2282 | return True |
| 2283 | |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 2284 | if self.failed_tids: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2285 | self.rq.state = runQueueFailed |
| 2286 | return True |
| 2287 | |
| 2288 | # Sanity Checks |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 2289 | err = self.summarise_scenequeue_errors() |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 2290 | for task in self.rqdata.runtaskentries: |
| 2291 | if task not in self.runq_buildable: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2292 | logger.error("Task %s never buildable!", task) |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 2293 | err = True |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2294 | elif task not in self.runq_running: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2295 | logger.error("Task %s never ran!", task) |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 2296 | err = True |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2297 | elif task not in self.runq_complete: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2298 | logger.error("Task %s never completed!", task) |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 2299 | err = True |
| 2300 | |
| 2301 | if err: |
| 2302 | self.rq.state = runQueueFailed |
| 2303 | else: |
| 2304 | self.rq.state = runQueueComplete |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2305 | |
| 2306 | return True |
| 2307 | |
Brad Bishop | c68388fc | 2019-08-26 01:33:31 -0400 | [diff] [blame] | 2308 | def filtermcdeps(self, task, mc, deps): |
Andrew Geissler | 99467da | 2019-02-25 18:54:23 -0600 | [diff] [blame] | 2309 | ret = set() |
Andrew Geissler | 99467da | 2019-02-25 18:54:23 -0600 | [diff] [blame] | 2310 | for dep in deps: |
Brad Bishop | c68388fc | 2019-08-26 01:33:31 -0400 | [diff] [blame] | 2311 | thismc = mc_from_tid(dep) |
| 2312 | if thismc != mc: |
Andrew Geissler | 99467da | 2019-02-25 18:54:23 -0600 | [diff] [blame] | 2313 | continue |
| 2314 | ret.add(dep) |
| 2315 | return ret |
| 2316 | |
Brad Bishop | a34c030 | 2019-09-23 22:34:48 -0400 | [diff] [blame] | 2317 | # We filter out multiconfig dependencies from taskdepdata we pass to the tasks |
Andrew Geissler | 99467da | 2019-02-25 18:54:23 -0600 | [diff] [blame] | 2318 | # as most code can't handle them |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2319 | def build_taskdepdata(self, task): |
| 2320 | taskdepdata = {} |
Brad Bishop | c68388fc | 2019-08-26 01:33:31 -0400 | [diff] [blame] | 2321 | mc = mc_from_tid(task) |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 2322 | next = self.rqdata.runtaskentries[task].depends.copy() |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2323 | next.add(task) |
Brad Bishop | c68388fc | 2019-08-26 01:33:31 -0400 | [diff] [blame] | 2324 | next = self.filtermcdeps(task, mc, next) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2325 | while next: |
| 2326 | additional = [] |
| 2327 | for revdep in next: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 2328 | (mc, fn, taskname, taskfn) = split_tid_mcfn(revdep) |
| 2329 | pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn] |
| 2330 | deps = self.rqdata.runtaskentries[revdep].depends |
| 2331 | provides = self.rqdata.dataCaches[mc].fn_provides[taskfn] |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 2332 | taskhash = self.rqdata.runtaskentries[revdep].hash |
Brad Bishop | 1932369 | 2019-04-05 15:28:33 -0400 | [diff] [blame] | 2333 | unihash = self.rqdata.runtaskentries[revdep].unihash |
Brad Bishop | c68388fc | 2019-08-26 01:33:31 -0400 | [diff] [blame] | 2334 | deps = self.filtermcdeps(task, mc, deps) |
Brad Bishop | 1932369 | 2019-04-05 15:28:33 -0400 | [diff] [blame] | 2335 | taskdepdata[revdep] = [pn, taskname, fn, deps, provides, taskhash, unihash] |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2336 | for revdep2 in deps: |
| 2337 | if revdep2 not in taskdepdata: |
| 2338 | additional.append(revdep2) |
| 2339 | next = additional |
| 2340 | |
| 2341 | #bb.note("Task %s: " % task + str(taskdepdata).replace("], ", "],\n")) |
| 2342 | return taskdepdata |
| 2343 | |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 2344 | def update_holdofftasks(self): |
Brad Bishop | c68388fc | 2019-08-26 01:33:31 -0400 | [diff] [blame] | 2345 | |
| 2346 | if not self.holdoff_need_update: |
| 2347 | return |
| 2348 | |
| 2349 | notcovered = set(self.scenequeue_notcovered) |
| 2350 | notcovered |= self.cantskip |
| 2351 | for tid in self.scenequeue_notcovered: |
| 2352 | notcovered |= self.sqdata.sq_covered_tasks[tid] |
| 2353 | notcovered |= self.sqdata.unskippable.difference(self.rqdata.runq_setscene_tids) |
| 2354 | notcovered.intersection_update(self.tasks_scenequeue_done) |
| 2355 | |
| 2356 | covered = set(self.scenequeue_covered) |
| 2357 | for tid in self.scenequeue_covered: |
| 2358 | covered |= self.sqdata.sq_covered_tasks[tid] |
| 2359 | covered.difference_update(notcovered) |
| 2360 | covered.intersection_update(self.tasks_scenequeue_done) |
| 2361 | |
| 2362 | for tid in notcovered | covered: |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 2363 | if not self.rqdata.runtaskentries[tid].depends: |
Brad Bishop | c68388fc | 2019-08-26 01:33:31 -0400 | [diff] [blame] | 2364 | self.setbuildable(tid) |
| 2365 | elif self.rqdata.runtaskentries[tid].depends.issubset(self.runq_complete): |
| 2366 | self.setbuildable(tid) |
| 2367 | |
| 2368 | self.tasks_covered = covered |
| 2369 | self.tasks_notcovered = notcovered |
| 2370 | |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 2371 | self.holdoff_tasks = set() |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2372 | |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 2373 | for tid in self.rqdata.runq_setscene_tids: |
| 2374 | if tid not in self.scenequeue_covered and tid not in self.scenequeue_notcovered: |
| 2375 | self.holdoff_tasks.add(tid) |
| 2376 | |
| 2377 | for tid in self.holdoff_tasks.copy(): |
| 2378 | for dep in self.sqdata.sq_covered_tasks[tid]: |
| 2379 | if dep not in self.runq_complete: |
| 2380 | self.holdoff_tasks.add(dep) |
| 2381 | |
Brad Bishop | c68388fc | 2019-08-26 01:33:31 -0400 | [diff] [blame] | 2382 | self.holdoff_need_update = False |
| 2383 | |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 2384 | def process_possible_migrations(self): |
| 2385 | |
| 2386 | changed = set() |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 2387 | toprocess = set() |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 2388 | for tid, unihash in self.updated_taskhash_queue.copy(): |
| 2389 | if tid in self.runq_running and tid not in self.runq_complete: |
| 2390 | continue |
| 2391 | |
| 2392 | self.updated_taskhash_queue.remove((tid, unihash)) |
| 2393 | |
| 2394 | if unihash != self.rqdata.runtaskentries[tid].unihash: |
Andrew Geissler | c926e17 | 2021-05-07 16:11:35 -0500 | [diff] [blame] | 2395 | # Make sure we rehash any other tasks with the same task hash that we're deferred against. |
| 2396 | torehash = [tid] |
| 2397 | for deftid in self.sq_deferred: |
| 2398 | if self.sq_deferred[deftid] == tid: |
| 2399 | torehash.append(deftid) |
| 2400 | for hashtid in torehash: |
| 2401 | hashequiv_logger.verbose("Task %s unihash changed to %s" % (hashtid, unihash)) |
| 2402 | self.rqdata.runtaskentries[hashtid].unihash = unihash |
| 2403 | bb.parse.siggen.set_unihash(hashtid, unihash) |
| 2404 | toprocess.add(hashtid) |
Andrew Geissler | 78b7279 | 2022-06-14 06:47:25 -0500 | [diff] [blame] | 2405 | if torehash: |
| 2406 | # Need to save after set_unihash above |
| 2407 | bb.parse.siggen.save_unitaskhashes() |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 2408 | |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 2409 | # Work out all tasks which depend upon these |
| 2410 | total = set() |
| 2411 | next = set() |
| 2412 | for p in toprocess: |
| 2413 | next |= self.rqdata.runtaskentries[p].revdeps |
| 2414 | while next: |
| 2415 | current = next.copy() |
| 2416 | total = total | next |
| 2417 | next = set() |
| 2418 | for ntid in current: |
| 2419 | next |= self.rqdata.runtaskentries[ntid].revdeps |
| 2420 | next.difference_update(total) |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 2421 | |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 2422 | # Now iterate those tasks in dependency order to regenerate their taskhash/unihash |
| 2423 | next = set() |
| 2424 | for p in total: |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 2425 | if not self.rqdata.runtaskentries[p].depends: |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 2426 | next.add(p) |
| 2427 | elif self.rqdata.runtaskentries[p].depends.isdisjoint(total): |
| 2428 | next.add(p) |
| 2429 | |
| 2430 | # When an item doesn't have dependencies in total, we can process it. Drop items from total when handled |
| 2431 | while next: |
| 2432 | current = next.copy() |
| 2433 | next = set() |
| 2434 | for tid in current: |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 2435 | if self.rqdata.runtaskentries[p].depends and not self.rqdata.runtaskentries[tid].depends.isdisjoint(total): |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 2436 | continue |
| 2437 | orighash = self.rqdata.runtaskentries[tid].hash |
Andrew Geissler | 517393d | 2023-01-13 08:55:19 -0600 | [diff] [blame] | 2438 | newhash = bb.parse.siggen.get_taskhash(tid, self.rqdata.runtaskentries[tid].depends, self.rqdata.dataCaches) |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 2439 | origuni = self.rqdata.runtaskentries[tid].unihash |
| 2440 | newuni = bb.parse.siggen.get_unihash(tid) |
| 2441 | # FIXME, need to check it can come from sstate at all for determinism? |
| 2442 | remapped = False |
| 2443 | if newuni == origuni: |
| 2444 | # Nothing to do, we match, skip code below |
| 2445 | remapped = True |
| 2446 | elif tid in self.scenequeue_covered or tid in self.sq_live: |
| 2447 | # Already ran this setscene task or it running. Report the new taskhash |
| 2448 | bb.parse.siggen.report_unihash_equiv(tid, newhash, origuni, newuni, self.rqdata.dataCaches) |
| 2449 | hashequiv_logger.verbose("Already covered setscene for %s so ignoring rehash (remap)" % (tid)) |
| 2450 | remapped = True |
| 2451 | |
| 2452 | if not remapped: |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 2453 | #logger.debug("Task %s hash changes: %s->%s %s->%s" % (tid, orighash, newhash, origuni, newuni)) |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 2454 | self.rqdata.runtaskentries[tid].hash = newhash |
| 2455 | self.rqdata.runtaskentries[tid].unihash = newuni |
| 2456 | changed.add(tid) |
| 2457 | |
| 2458 | next |= self.rqdata.runtaskentries[tid].revdeps |
| 2459 | total.remove(tid) |
| 2460 | next.intersection_update(total) |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 2461 | |
| 2462 | if changed: |
| 2463 | for mc in self.rq.worker: |
| 2464 | self.rq.worker[mc].process.stdin.write(b"<newtaskhashes>" + pickle.dumps(bb.parse.siggen.get_taskhashes()) + b"</newtaskhashes>") |
| 2465 | for mc in self.rq.fakeworker: |
| 2466 | self.rq.fakeworker[mc].process.stdin.write(b"<newtaskhashes>" + pickle.dumps(bb.parse.siggen.get_taskhashes()) + b"</newtaskhashes>") |
| 2467 | |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 2468 | hashequiv_logger.debug(pprint.pformat("Tasks changed:\n%s" % (changed))) |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 2469 | |
| 2470 | for tid in changed: |
| 2471 | if tid not in self.rqdata.runq_setscene_tids: |
| 2472 | continue |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 2473 | if tid not in self.pending_migrations: |
| 2474 | self.pending_migrations.add(tid) |
| 2475 | |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 2476 | update_tasks = [] |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 2477 | for tid in self.pending_migrations.copy(): |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 2478 | if tid in self.runq_running or tid in self.sq_live: |
Brad Bishop | 6dbb316 | 2019-11-25 09:41:34 -0500 | [diff] [blame] | 2479 | # Too late, task already running, not much we can do now |
| 2480 | self.pending_migrations.remove(tid) |
| 2481 | continue |
| 2482 | |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 2483 | valid = True |
| 2484 | # Check no tasks this covers are running |
| 2485 | for dep in self.sqdata.sq_covered_tasks[tid]: |
| 2486 | if dep in self.runq_running and dep not in self.runq_complete: |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 2487 | hashequiv_logger.debug2("Task %s is running which blocks setscene for %s from running" % (dep, tid)) |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 2488 | valid = False |
| 2489 | break |
| 2490 | if not valid: |
| 2491 | continue |
| 2492 | |
| 2493 | self.pending_migrations.remove(tid) |
Brad Bishop | c68388fc | 2019-08-26 01:33:31 -0400 | [diff] [blame] | 2494 | changed = True |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 2495 | |
| 2496 | if tid in self.tasks_scenequeue_done: |
| 2497 | self.tasks_scenequeue_done.remove(tid) |
| 2498 | for dep in self.sqdata.sq_covered_tasks[tid]: |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 2499 | if dep in self.runq_complete and dep not in self.runq_tasksrun: |
Andrew Geissler | 7e0e3c0 | 2022-02-25 20:34:39 +0000 | [diff] [blame] | 2500 | bb.error("Task %s marked as completed but now needing to rerun? Halting build." % dep) |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 2501 | self.failed_tids.append(tid) |
| 2502 | self.rq.state = runQueueCleanUp |
| 2503 | return |
| 2504 | |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 2505 | if dep not in self.runq_complete: |
| 2506 | if dep in self.tasks_scenequeue_done and dep not in self.sqdata.unskippable: |
| 2507 | self.tasks_scenequeue_done.remove(dep) |
| 2508 | |
| 2509 | if tid in self.sq_buildable: |
| 2510 | self.sq_buildable.remove(tid) |
| 2511 | if tid in self.sq_running: |
| 2512 | self.sq_running.remove(tid) |
Andrew Geissler | 517393d | 2023-01-13 08:55:19 -0600 | [diff] [blame] | 2513 | if tid in self.sqdata.outrightfail: |
| 2514 | self.sqdata.outrightfail.remove(tid) |
| 2515 | if tid in self.scenequeue_notcovered: |
| 2516 | self.scenequeue_notcovered.remove(tid) |
| 2517 | if tid in self.scenequeue_covered: |
| 2518 | self.scenequeue_covered.remove(tid) |
| 2519 | if tid in self.scenequeue_notneeded: |
| 2520 | self.scenequeue_notneeded.remove(tid) |
| 2521 | |
| 2522 | (mc, fn, taskname, taskfn) = split_tid_mcfn(tid) |
| 2523 | self.sqdata.stamps[tid] = bb.parse.siggen.stampfile_mcfn(taskname, taskfn, extrainfo=False) |
| 2524 | |
| 2525 | if tid in self.stampcache: |
| 2526 | del self.stampcache[tid] |
| 2527 | |
| 2528 | if tid in self.build_stamps: |
| 2529 | del self.build_stamps[tid] |
| 2530 | |
| 2531 | update_tasks.append(tid) |
| 2532 | |
| 2533 | update_tasks2 = [] |
| 2534 | for tid in update_tasks: |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 2535 | harddepfail = False |
| 2536 | for t in self.sqdata.sq_harddeps: |
| 2537 | if tid in self.sqdata.sq_harddeps[t] and t in self.scenequeue_notcovered: |
| 2538 | harddepfail = True |
| 2539 | break |
| 2540 | if not harddepfail and self.sqdata.sq_revdeps[tid].issubset(self.scenequeue_covered | self.scenequeue_notcovered): |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 2541 | if tid not in self.sq_buildable: |
| 2542 | self.sq_buildable.add(tid) |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 2543 | if not self.sqdata.sq_revdeps[tid]: |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 2544 | self.sq_buildable.add(tid) |
| 2545 | |
Andrew Geissler | 517393d | 2023-01-13 08:55:19 -0600 | [diff] [blame] | 2546 | update_tasks2.append((tid, harddepfail, tid in self.sqdata.valid)) |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 2547 | |
Andrew Geissler | 517393d | 2023-01-13 08:55:19 -0600 | [diff] [blame] | 2548 | if update_tasks2: |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 2549 | self.sqdone = False |
Patrick Williams | 92b42cb | 2022-09-03 06:53:57 -0500 | [diff] [blame] | 2550 | for mc in sorted(self.sqdata.multiconfigs): |
Andrew Geissler | 517393d | 2023-01-13 08:55:19 -0600 | [diff] [blame] | 2551 | for tid in sorted([t[0] for t in update_tasks2]): |
Patrick Williams | 92b42cb | 2022-09-03 06:53:57 -0500 | [diff] [blame] | 2552 | if mc_from_tid(tid) != mc: |
| 2553 | continue |
| 2554 | h = pending_hash_index(tid, self.rqdata) |
| 2555 | if h in self.sqdata.hashes and tid != self.sqdata.hashes[h]: |
| 2556 | self.sq_deferred[tid] = self.sqdata.hashes[h] |
| 2557 | bb.note("Deferring %s after %s" % (tid, self.sqdata.hashes[h])) |
Andrew Geissler | 517393d | 2023-01-13 08:55:19 -0600 | [diff] [blame] | 2558 | update_scenequeue_data([t[0] for t in update_tasks2], self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self, summary=False) |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 2559 | |
Andrew Geissler | 517393d | 2023-01-13 08:55:19 -0600 | [diff] [blame] | 2560 | for (tid, harddepfail, origvalid) in update_tasks2: |
Brad Bishop | 1d80a2e | 2019-11-15 16:35:03 -0500 | [diff] [blame] | 2561 | if tid in self.sqdata.valid and not origvalid: |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 2562 | hashequiv_logger.verbose("Setscene task %s became valid" % tid) |
| 2563 | if harddepfail: |
Andrew Geissler | 517393d | 2023-01-13 08:55:19 -0600 | [diff] [blame] | 2564 | logger.debug2("%s has an unavailable hard dependency so skipping" % (tid)) |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 2565 | self.sq_task_failoutright(tid) |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 2566 | |
| 2567 | if changed: |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 2568 | self.stats.updateCovered(len(self.scenequeue_covered), len(self.scenequeue_notcovered)) |
Brad Bishop | c68388fc | 2019-08-26 01:33:31 -0400 | [diff] [blame] | 2569 | self.holdoff_need_update = True |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2570 | |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2571 | def scenequeue_updatecounters(self, task, fail=False): |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 2572 | |
| 2573 | for dep in sorted(self.sqdata.sq_deps[task]): |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2574 | if fail and task in self.sqdata.sq_harddeps and dep in self.sqdata.sq_harddeps[task]: |
Andrew Geissler | 95ac1b8 | 2021-03-31 14:34:31 -0500 | [diff] [blame] | 2575 | if dep in self.scenequeue_covered or dep in self.scenequeue_notcovered: |
| 2576 | # dependency could be already processed, e.g. noexec setscene task |
| 2577 | continue |
Andrew Geissler | 3b8a17c | 2021-04-15 15:55:55 -0500 | [diff] [blame] | 2578 | noexec, stamppresent = check_setscene_stamps(dep, self.rqdata, self.rq, self.stampcache) |
| 2579 | if noexec or stamppresent: |
| 2580 | continue |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 2581 | logger.debug2("%s was unavailable and is a hard dependency of %s so skipping" % (task, dep)) |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2582 | self.sq_task_failoutright(dep) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2583 | continue |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 2584 | if self.sqdata.sq_revdeps[dep].issubset(self.scenequeue_covered | self.scenequeue_notcovered): |
| 2585 | if dep not in self.sq_buildable: |
| 2586 | self.sq_buildable.add(dep) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2587 | |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2588 | next = set([task]) |
| 2589 | while next: |
| 2590 | new = set() |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 2591 | for t in sorted(next): |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2592 | self.tasks_scenequeue_done.add(t) |
| 2593 | # Look down the dependency chain for non-setscene things which this task depends on |
| 2594 | # and mark as 'done' |
| 2595 | for dep in self.rqdata.runtaskentries[t].depends: |
| 2596 | if dep in self.rqdata.runq_setscene_tids or dep in self.tasks_scenequeue_done: |
| 2597 | continue |
| 2598 | if self.rqdata.runtaskentries[dep].revdeps.issubset(self.tasks_scenequeue_done): |
| 2599 | new.add(dep) |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2600 | next = new |
| 2601 | |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 2602 | self.stats.updateCovered(len(self.scenequeue_covered), len(self.scenequeue_notcovered)) |
Brad Bishop | c68388fc | 2019-08-26 01:33:31 -0400 | [diff] [blame] | 2603 | self.holdoff_need_update = True |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2604 | |
| 2605 | def sq_task_completeoutright(self, task): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2606 | """ |
| 2607 | Mark a task as completed |
| 2608 | Look at the reverse dependencies and mark any task with |
| 2609 | completed dependencies as buildable |
| 2610 | """ |
| 2611 | |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 2612 | logger.debug('Found task %s which could be accelerated', task) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2613 | self.scenequeue_covered.add(task) |
| 2614 | self.scenequeue_updatecounters(task) |
| 2615 | |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2616 | def sq_check_taskfail(self, task): |
Andrew Geissler | 7e0e3c0 | 2022-02-25 20:34:39 +0000 | [diff] [blame] | 2617 | if self.rqdata.setscene_ignore_tasks is not None: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 2618 | realtask = task.split('_setscene')[0] |
Brad Bishop | 37a0e4d | 2017-12-04 01:01:44 -0500 | [diff] [blame] | 2619 | (mc, fn, taskname, taskfn) = split_tid_mcfn(realtask) |
| 2620 | pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn] |
Andrew Geissler | 7e0e3c0 | 2022-02-25 20:34:39 +0000 | [diff] [blame] | 2621 | if not check_setscene_enforce_ignore_tasks(pn, taskname, self.rqdata.setscene_ignore_tasks): |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 2622 | logger.error('Task %s.%s failed' % (pn, taskname + "_setscene")) |
| 2623 | self.rq.state = runQueueCleanUp |
| 2624 | |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2625 | def sq_task_complete(self, task): |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 2626 | bb.event.fire(sceneQueueTaskCompleted(task, self.stats, self.rq), self.cfgData) |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2627 | self.sq_task_completeoutright(task) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2628 | |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2629 | def sq_task_fail(self, task, result): |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 2630 | bb.event.fire(sceneQueueTaskFailed(task, self.stats, result, self), self.cfgData) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2631 | self.scenequeue_notcovered.add(task) |
| 2632 | self.scenequeue_updatecounters(task, True) |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2633 | self.sq_check_taskfail(task) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2634 | |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2635 | def sq_task_failoutright(self, task): |
| 2636 | self.sq_running.add(task) |
| 2637 | self.sq_buildable.add(task) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2638 | self.scenequeue_notcovered.add(task) |
| 2639 | self.scenequeue_updatecounters(task, True) |
| 2640 | |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2641 | def sq_task_skip(self, task): |
| 2642 | self.sq_running.add(task) |
| 2643 | self.sq_buildable.add(task) |
| 2644 | self.sq_task_completeoutright(task) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2645 | |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2646 | def sq_build_taskdepdata(self, task): |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 2647 | def getsetscenedeps(tid): |
| 2648 | deps = set() |
| 2649 | (mc, fn, taskname, _) = split_tid_mcfn(tid) |
| 2650 | realtid = tid + "_setscene" |
| 2651 | idepends = self.rqdata.taskData[mc].taskentries[realtid].idepends |
| 2652 | for (depname, idependtask) in idepends: |
| 2653 | if depname not in self.rqdata.taskData[mc].build_targets: |
| 2654 | continue |
| 2655 | |
| 2656 | depfn = self.rqdata.taskData[mc].build_targets[depname][0] |
| 2657 | if depfn is None: |
| 2658 | continue |
| 2659 | deptid = depfn + ":" + idependtask.replace("_setscene", "") |
| 2660 | deps.add(deptid) |
| 2661 | return deps |
| 2662 | |
| 2663 | taskdepdata = {} |
| 2664 | next = getsetscenedeps(task) |
| 2665 | next.add(task) |
| 2666 | while next: |
| 2667 | additional = [] |
| 2668 | for revdep in next: |
| 2669 | (mc, fn, taskname, taskfn) = split_tid_mcfn(revdep) |
| 2670 | pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn] |
| 2671 | deps = getsetscenedeps(revdep) |
| 2672 | provides = self.rqdata.dataCaches[mc].fn_provides[taskfn] |
| 2673 | taskhash = self.rqdata.runtaskentries[revdep].hash |
Brad Bishop | 1932369 | 2019-04-05 15:28:33 -0400 | [diff] [blame] | 2674 | unihash = self.rqdata.runtaskentries[revdep].unihash |
| 2675 | taskdepdata[revdep] = [pn, taskname, fn, deps, provides, taskhash, unihash] |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 2676 | for revdep2 in deps: |
| 2677 | if revdep2 not in taskdepdata: |
| 2678 | additional.append(revdep2) |
| 2679 | next = additional |
| 2680 | |
| 2681 | #bb.note("Task %s: " % task + str(taskdepdata).replace("], ", "],\n")) |
| 2682 | return taskdepdata |
| 2683 | |
Andrew Geissler | 7e0e3c0 | 2022-02-25 20:34:39 +0000 | [diff] [blame] | 2684 | def check_setscene_ignore_tasks(self, tid): |
| 2685 | # Check task that is going to run against the ignore tasks list |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2686 | (mc, fn, taskname, taskfn) = split_tid_mcfn(tid) |
| 2687 | # Ignore covered tasks |
| 2688 | if tid in self.tasks_covered: |
| 2689 | return False |
| 2690 | # Ignore stamped tasks |
| 2691 | if self.rq.check_stamp_task(tid, taskname, cache=self.stampcache): |
| 2692 | return False |
| 2693 | # Ignore noexec tasks |
| 2694 | taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn] |
| 2695 | if 'noexec' in taskdep and taskname in taskdep['noexec']: |
| 2696 | return False |
| 2697 | |
| 2698 | pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn] |
Andrew Geissler | 7e0e3c0 | 2022-02-25 20:34:39 +0000 | [diff] [blame] | 2699 | if not check_setscene_enforce_ignore_tasks(pn, taskname, self.rqdata.setscene_ignore_tasks): |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2700 | if tid in self.rqdata.runq_setscene_tids: |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 2701 | msg = ['Task %s.%s attempted to execute unexpectedly and should have been setscened' % (pn, taskname)] |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2702 | else: |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 2703 | msg = ['Task %s.%s attempted to execute unexpectedly' % (pn, taskname)] |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 2704 | for t in self.scenequeue_notcovered: |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 2705 | msg.append("\nTask %s, unihash %s, taskhash %s" % (t, self.rqdata.runtaskentries[t].unihash, self.rqdata.runtaskentries[t].hash)) |
| 2706 | msg.append('\nThis is usually due to missing setscene tasks. Those missing in this build were: %s' % pprint.pformat(self.scenequeue_notcovered)) |
| 2707 | logger.error("".join(msg)) |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2708 | return True |
| 2709 | return False |
| 2710 | |
| 2711 | class SQData(object): |
| 2712 | def __init__(self): |
| 2713 | # SceneQueue dependencies |
| 2714 | self.sq_deps = {} |
| 2715 | # SceneQueue reverse dependencies |
| 2716 | self.sq_revdeps = {} |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2717 | # Injected inter-setscene task dependencies |
| 2718 | self.sq_harddeps = {} |
| 2719 | # Cache of stamp files so duplicates can't run in parallel |
| 2720 | self.stamps = {} |
| 2721 | # Setscene tasks directly depended upon by the build |
| 2722 | self.unskippable = set() |
| 2723 | # List of setscene tasks which aren't present |
| 2724 | self.outrightfail = set() |
| 2725 | # A list of normal tasks a setscene task covers |
| 2726 | self.sq_covered_tasks = {} |
| 2727 | |
| 2728 | def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq): |
| 2729 | |
| 2730 | sq_revdeps = {} |
| 2731 | sq_revdeps_squash = {} |
| 2732 | sq_collated_deps = {} |
| 2733 | |
| 2734 | # We need to construct a dependency graph for the setscene functions. Intermediate |
| 2735 | # dependencies between the setscene tasks only complicate the code. This code |
| 2736 | # therefore aims to collapse the huge runqueue dependency tree into a smaller one |
| 2737 | # only containing the setscene functions. |
| 2738 | |
| 2739 | rqdata.init_progress_reporter.next_stage() |
| 2740 | |
| 2741 | # First process the chains up to the first setscene task. |
| 2742 | endpoints = {} |
| 2743 | for tid in rqdata.runtaskentries: |
| 2744 | sq_revdeps[tid] = copy.copy(rqdata.runtaskentries[tid].revdeps) |
| 2745 | sq_revdeps_squash[tid] = set() |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 2746 | if not sq_revdeps[tid] and tid not in rqdata.runq_setscene_tids: |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2747 | #bb.warn("Added endpoint %s" % (tid)) |
| 2748 | endpoints[tid] = set() |
| 2749 | |
| 2750 | rqdata.init_progress_reporter.next_stage() |
| 2751 | |
| 2752 | # Secondly process the chains between setscene tasks. |
| 2753 | for tid in rqdata.runq_setscene_tids: |
| 2754 | sq_collated_deps[tid] = set() |
| 2755 | #bb.warn("Added endpoint 2 %s" % (tid)) |
| 2756 | for dep in rqdata.runtaskentries[tid].depends: |
| 2757 | if tid in sq_revdeps[dep]: |
| 2758 | sq_revdeps[dep].remove(tid) |
| 2759 | if dep not in endpoints: |
| 2760 | endpoints[dep] = set() |
| 2761 | #bb.warn(" Added endpoint 3 %s" % (dep)) |
| 2762 | endpoints[dep].add(tid) |
| 2763 | |
| 2764 | rqdata.init_progress_reporter.next_stage() |
| 2765 | |
| 2766 | def process_endpoints(endpoints): |
| 2767 | newendpoints = {} |
| 2768 | for point, task in endpoints.items(): |
| 2769 | tasks = set() |
| 2770 | if task: |
| 2771 | tasks |= task |
| 2772 | if sq_revdeps_squash[point]: |
| 2773 | tasks |= sq_revdeps_squash[point] |
| 2774 | if point not in rqdata.runq_setscene_tids: |
| 2775 | for t in tasks: |
| 2776 | sq_collated_deps[t].add(point) |
| 2777 | sq_revdeps_squash[point] = set() |
| 2778 | if point in rqdata.runq_setscene_tids: |
| 2779 | sq_revdeps_squash[point] = tasks |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2780 | continue |
| 2781 | for dep in rqdata.runtaskentries[point].depends: |
| 2782 | if point in sq_revdeps[dep]: |
| 2783 | sq_revdeps[dep].remove(point) |
| 2784 | if tasks: |
| 2785 | sq_revdeps_squash[dep] |= tasks |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 2786 | if not sq_revdeps[dep] and dep not in rqdata.runq_setscene_tids: |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2787 | newendpoints[dep] = task |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 2788 | if newendpoints: |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2789 | process_endpoints(newendpoints) |
| 2790 | |
| 2791 | process_endpoints(endpoints) |
| 2792 | |
| 2793 | rqdata.init_progress_reporter.next_stage() |
| 2794 | |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 2795 | # Build a list of tasks which are "unskippable" |
| 2796 | # These are direct endpoints referenced by the build upto and including setscene tasks |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2797 | # Take the build endpoints (no revdeps) and find the sstate tasks they depend upon |
| 2798 | new = True |
| 2799 | for tid in rqdata.runtaskentries: |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 2800 | if not rqdata.runtaskentries[tid].revdeps: |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2801 | sqdata.unskippable.add(tid) |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 2802 | sqdata.unskippable |= sqrq.cantskip |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2803 | while new: |
| 2804 | new = False |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 2805 | orig = sqdata.unskippable.copy() |
| 2806 | for tid in sorted(orig, reverse=True): |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2807 | if tid in rqdata.runq_setscene_tids: |
| 2808 | continue |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 2809 | if not rqdata.runtaskentries[tid].depends: |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2810 | # These are tasks which have no setscene tasks in their chain, need to mark as directly buildable |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2811 | sqrq.setbuildable(tid) |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2812 | sqdata.unskippable |= rqdata.runtaskentries[tid].depends |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 2813 | if sqdata.unskippable != orig: |
| 2814 | new = True |
| 2815 | |
| 2816 | sqrq.tasks_scenequeue_done |= sqdata.unskippable.difference(rqdata.runq_setscene_tids) |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2817 | |
| 2818 | rqdata.init_progress_reporter.next_stage(len(rqdata.runtaskentries)) |
| 2819 | |
| 2820 | # Sanity check all dependencies could be changed to setscene task references |
| 2821 | for taskcounter, tid in enumerate(rqdata.runtaskentries): |
| 2822 | if tid in rqdata.runq_setscene_tids: |
| 2823 | pass |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 2824 | elif sq_revdeps_squash[tid]: |
Andrew Geissler | 7e0e3c0 | 2022-02-25 20:34:39 +0000 | [diff] [blame] | 2825 | bb.msg.fatal("RunQueue", "Something went badly wrong during scenequeue generation, halting. Please report this problem.") |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2826 | else: |
| 2827 | del sq_revdeps_squash[tid] |
| 2828 | rqdata.init_progress_reporter.update(taskcounter) |
| 2829 | |
| 2830 | rqdata.init_progress_reporter.next_stage() |
| 2831 | |
| 2832 | # Resolve setscene inter-task dependencies |
| 2833 | # e.g. do_sometask_setscene[depends] = "targetname:do_someothertask_setscene" |
| 2834 | # Note that anything explicitly depended upon will have its reverse dependencies removed to avoid circular dependencies |
| 2835 | for tid in rqdata.runq_setscene_tids: |
| 2836 | (mc, fn, taskname, taskfn) = split_tid_mcfn(tid) |
| 2837 | realtid = tid + "_setscene" |
| 2838 | idepends = rqdata.taskData[mc].taskentries[realtid].idepends |
Andrew Geissler | 517393d | 2023-01-13 08:55:19 -0600 | [diff] [blame] | 2839 | sqdata.stamps[tid] = bb.parse.siggen.stampfile_mcfn(taskname, taskfn, extrainfo=False) |
| 2840 | |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2841 | for (depname, idependtask) in idepends: |
| 2842 | |
| 2843 | if depname not in rqdata.taskData[mc].build_targets: |
| 2844 | continue |
| 2845 | |
| 2846 | depfn = rqdata.taskData[mc].build_targets[depname][0] |
| 2847 | if depfn is None: |
| 2848 | continue |
| 2849 | deptid = depfn + ":" + idependtask.replace("_setscene", "") |
| 2850 | if deptid not in rqdata.runtaskentries: |
| 2851 | bb.msg.fatal("RunQueue", "Task %s depends upon non-existent task %s:%s" % (realtid, depfn, idependtask)) |
| 2852 | |
| 2853 | if not deptid in sqdata.sq_harddeps: |
| 2854 | sqdata.sq_harddeps[deptid] = set() |
| 2855 | sqdata.sq_harddeps[deptid].add(tid) |
| 2856 | |
| 2857 | sq_revdeps_squash[tid].add(deptid) |
| 2858 | # Have to zero this to avoid circular dependencies |
| 2859 | sq_revdeps_squash[deptid] = set() |
| 2860 | |
| 2861 | rqdata.init_progress_reporter.next_stage() |
| 2862 | |
| 2863 | for task in sqdata.sq_harddeps: |
| 2864 | for dep in sqdata.sq_harddeps[task]: |
| 2865 | sq_revdeps_squash[dep].add(task) |
| 2866 | |
| 2867 | rqdata.init_progress_reporter.next_stage() |
| 2868 | |
| 2869 | #for tid in sq_revdeps_squash: |
| 2870 | # data = "" |
| 2871 | # for dep in sq_revdeps_squash[tid]: |
| 2872 | # data = data + "\n %s" % dep |
| 2873 | # bb.warn("Task %s_setscene: is %s " % (tid, data)) |
| 2874 | |
| 2875 | sqdata.sq_revdeps = sq_revdeps_squash |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2876 | sqdata.sq_covered_tasks = sq_collated_deps |
| 2877 | |
| 2878 | # Build reverse version of revdeps to populate deps structure |
| 2879 | for tid in sqdata.sq_revdeps: |
| 2880 | sqdata.sq_deps[tid] = set() |
| 2881 | for tid in sqdata.sq_revdeps: |
| 2882 | for dep in sqdata.sq_revdeps[tid]: |
| 2883 | sqdata.sq_deps[dep].add(tid) |
| 2884 | |
| 2885 | rqdata.init_progress_reporter.next_stage() |
| 2886 | |
Brad Bishop | 00e122a | 2019-10-05 11:10:57 -0400 | [diff] [blame] | 2887 | sqdata.multiconfigs = set() |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2888 | for tid in sqdata.sq_revdeps: |
Brad Bishop | 00e122a | 2019-10-05 11:10:57 -0400 | [diff] [blame] | 2889 | sqdata.multiconfigs.add(mc_from_tid(tid)) |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 2890 | if not sqdata.sq_revdeps[tid]: |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2891 | sqrq.sq_buildable.add(tid) |
| 2892 | |
| 2893 | rqdata.init_progress_reporter.finish() |
| 2894 | |
Brad Bishop | 00e122a | 2019-10-05 11:10:57 -0400 | [diff] [blame] | 2895 | sqdata.noexec = set() |
| 2896 | sqdata.stamppresent = set() |
| 2897 | sqdata.valid = set() |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2898 | |
Patrick Williams | 213cb26 | 2021-08-07 19:21:33 -0500 | [diff] [blame] | 2899 | sqdata.hashes = {} |
| 2900 | sqrq.sq_deferred = {} |
| 2901 | for mc in sorted(sqdata.multiconfigs): |
| 2902 | for tid in sorted(sqdata.sq_revdeps): |
| 2903 | if mc_from_tid(tid) != mc: |
| 2904 | continue |
| 2905 | h = pending_hash_index(tid, rqdata) |
| 2906 | if h not in sqdata.hashes: |
| 2907 | sqdata.hashes[h] = tid |
| 2908 | else: |
| 2909 | sqrq.sq_deferred[tid] = sqdata.hashes[h] |
| 2910 | bb.note("Deferring %s after %s" % (tid, sqdata.hashes[h])) |
| 2911 | |
Brad Bishop | 1d80a2e | 2019-11-15 16:35:03 -0500 | [diff] [blame] | 2912 | update_scenequeue_data(sqdata.sq_revdeps, sqdata, rqdata, rq, cooker, stampcache, sqrq, summary=True) |
Brad Bishop | 00e122a | 2019-10-05 11:10:57 -0400 | [diff] [blame] | 2913 | |
Andrew Geissler | 95ac1b8 | 2021-03-31 14:34:31 -0500 | [diff] [blame] | 2914 | # Compute a list of 'stale' sstate tasks where the current hash does not match the one |
| 2915 | # in any stamp files. Pass the list out to metadata as an event. |
| 2916 | found = {} |
| 2917 | for tid in rqdata.runq_setscene_tids: |
| 2918 | (mc, fn, taskname, taskfn) = split_tid_mcfn(tid) |
Andrew Geissler | 517393d | 2023-01-13 08:55:19 -0600 | [diff] [blame] | 2919 | stamps = bb.build.find_stale_stamps(taskname, taskfn) |
Andrew Geissler | 95ac1b8 | 2021-03-31 14:34:31 -0500 | [diff] [blame] | 2920 | if stamps: |
| 2921 | if mc not in found: |
| 2922 | found[mc] = {} |
| 2923 | found[mc][tid] = stamps |
| 2924 | for mc in found: |
| 2925 | event = bb.event.StaleSetSceneTasks(found[mc]) |
| 2926 | bb.event.fire(event, cooker.databuilder.mcdata[mc]) |
| 2927 | |
Andrew Geissler | 3b8a17c | 2021-04-15 15:55:55 -0500 | [diff] [blame] | 2928 | def check_setscene_stamps(tid, rqdata, rq, stampcache, noexecstamp=False): |
| 2929 | |
| 2930 | (mc, fn, taskname, taskfn) = split_tid_mcfn(tid) |
| 2931 | |
| 2932 | taskdep = rqdata.dataCaches[mc].task_deps[taskfn] |
| 2933 | |
| 2934 | if 'noexec' in taskdep and taskname in taskdep['noexec']: |
Andrew Geissler | 517393d | 2023-01-13 08:55:19 -0600 | [diff] [blame] | 2935 | bb.build.make_stamp_mcfn(taskname + "_setscene", taskfn) |
Andrew Geissler | 3b8a17c | 2021-04-15 15:55:55 -0500 | [diff] [blame] | 2936 | return True, False |
| 2937 | |
| 2938 | if rq.check_stamp_task(tid, taskname + "_setscene", cache=stampcache): |
| 2939 | logger.debug2('Setscene stamp current for task %s', tid) |
| 2940 | return False, True |
| 2941 | |
| 2942 | if rq.check_stamp_task(tid, taskname, recurse = True, cache=stampcache): |
| 2943 | logger.debug2('Normal stamp current for task %s', tid) |
| 2944 | return False, True |
| 2945 | |
| 2946 | return False, False |
| 2947 | |
Brad Bishop | 1d80a2e | 2019-11-15 16:35:03 -0500 | [diff] [blame] | 2948 | def update_scenequeue_data(tids, sqdata, rqdata, rq, cooker, stampcache, sqrq, summary=True): |
Brad Bishop | 00e122a | 2019-10-05 11:10:57 -0400 | [diff] [blame] | 2949 | |
| 2950 | tocheck = set() |
| 2951 | |
| 2952 | for tid in sorted(tids): |
| 2953 | if tid in sqdata.stamppresent: |
| 2954 | sqdata.stamppresent.remove(tid) |
| 2955 | if tid in sqdata.valid: |
| 2956 | sqdata.valid.remove(tid) |
Andrew Geissler | c926e17 | 2021-05-07 16:11:35 -0500 | [diff] [blame] | 2957 | if tid in sqdata.outrightfail: |
| 2958 | sqdata.outrightfail.remove(tid) |
Brad Bishop | 00e122a | 2019-10-05 11:10:57 -0400 | [diff] [blame] | 2959 | |
Andrew Geissler | 3b8a17c | 2021-04-15 15:55:55 -0500 | [diff] [blame] | 2960 | noexec, stamppresent = check_setscene_stamps(tid, rqdata, rq, stampcache, noexecstamp=True) |
Brad Bishop | 00e122a | 2019-10-05 11:10:57 -0400 | [diff] [blame] | 2961 | |
Andrew Geissler | 3b8a17c | 2021-04-15 15:55:55 -0500 | [diff] [blame] | 2962 | if noexec: |
Brad Bishop | 00e122a | 2019-10-05 11:10:57 -0400 | [diff] [blame] | 2963 | sqdata.noexec.add(tid) |
| 2964 | sqrq.sq_task_skip(tid) |
Andrew Geissler | 517393d | 2023-01-13 08:55:19 -0600 | [diff] [blame] | 2965 | logger.debug2("%s is noexec so skipping setscene" % (tid)) |
Brad Bishop | 00e122a | 2019-10-05 11:10:57 -0400 | [diff] [blame] | 2966 | continue |
| 2967 | |
Andrew Geissler | 3b8a17c | 2021-04-15 15:55:55 -0500 | [diff] [blame] | 2968 | if stamppresent: |
Brad Bishop | 00e122a | 2019-10-05 11:10:57 -0400 | [diff] [blame] | 2969 | sqdata.stamppresent.add(tid) |
| 2970 | sqrq.sq_task_skip(tid) |
Andrew Geissler | 517393d | 2023-01-13 08:55:19 -0600 | [diff] [blame] | 2971 | logger.debug2("%s has a valid stamp, skipping" % (tid)) |
Brad Bishop | 00e122a | 2019-10-05 11:10:57 -0400 | [diff] [blame] | 2972 | continue |
| 2973 | |
| 2974 | tocheck.add(tid) |
| 2975 | |
Brad Bishop | 1d80a2e | 2019-11-15 16:35:03 -0500 | [diff] [blame] | 2976 | sqdata.valid |= rq.validate_hashes(tocheck, cooker.data, len(sqdata.stamppresent), False, summary=summary) |
Brad Bishop | 00e122a | 2019-10-05 11:10:57 -0400 | [diff] [blame] | 2977 | |
Patrick Williams | 213cb26 | 2021-08-07 19:21:33 -0500 | [diff] [blame] | 2978 | for tid in tids: |
| 2979 | if tid in sqdata.stamppresent: |
| 2980 | continue |
| 2981 | if tid in sqdata.valid: |
| 2982 | continue |
| 2983 | if tid in sqdata.noexec: |
| 2984 | continue |
| 2985 | if tid in sqrq.scenequeue_covered: |
| 2986 | continue |
| 2987 | if tid in sqrq.scenequeue_notcovered: |
| 2988 | continue |
| 2989 | if tid in sqrq.sq_deferred: |
| 2990 | continue |
| 2991 | sqdata.outrightfail.add(tid) |
Andrew Geissler | 517393d | 2023-01-13 08:55:19 -0600 | [diff] [blame] | 2992 | logger.debug2("%s already handled (fallthrough), skipping" % (tid)) |
Brad Bishop | 96ff198 | 2019-08-19 13:50:42 -0400 | [diff] [blame] | 2993 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2994 | class TaskFailure(Exception): |
| 2995 | """ |
| 2996 | Exception raised when a task in a runqueue fails |
| 2997 | """ |
| 2998 | def __init__(self, x): |
| 2999 | self.args = x |
| 3000 | |
| 3001 | |
| 3002 | class runQueueExitWait(bb.event.Event): |
| 3003 | """ |
| 3004 | Event when waiting for task processes to exit |
| 3005 | """ |
| 3006 | |
| 3007 | def __init__(self, remain): |
| 3008 | self.remain = remain |
| 3009 | self.message = "Waiting for %s active tasks to finish" % remain |
| 3010 | bb.event.Event.__init__(self) |
| 3011 | |
| 3012 | class runQueueEvent(bb.event.Event): |
| 3013 | """ |
| 3014 | Base runQueue event class |
| 3015 | """ |
| 3016 | def __init__(self, task, stats, rq): |
| 3017 | self.taskid = task |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 3018 | self.taskstring = task |
| 3019 | self.taskname = taskname_from_tid(task) |
| 3020 | self.taskfile = fn_from_tid(task) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 3021 | self.taskhash = rq.rqdata.get_task_hash(task) |
| 3022 | self.stats = stats.copy() |
| 3023 | bb.event.Event.__init__(self) |
| 3024 | |
| 3025 | class sceneQueueEvent(runQueueEvent): |
| 3026 | """ |
| 3027 | Base sceneQueue event class |
| 3028 | """ |
| 3029 | def __init__(self, task, stats, rq, noexec=False): |
| 3030 | runQueueEvent.__init__(self, task, stats, rq) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 3031 | self.taskstring = task + "_setscene" |
| 3032 | self.taskname = taskname_from_tid(task) + "_setscene" |
| 3033 | self.taskfile = fn_from_tid(task) |
| 3034 | self.taskhash = rq.rqdata.get_task_hash(task) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 3035 | |
| 3036 | class runQueueTaskStarted(runQueueEvent): |
| 3037 | """ |
| 3038 | Event notifying a task was started |
| 3039 | """ |
| 3040 | def __init__(self, task, stats, rq, noexec=False): |
| 3041 | runQueueEvent.__init__(self, task, stats, rq) |
| 3042 | self.noexec = noexec |
| 3043 | |
| 3044 | class sceneQueueTaskStarted(sceneQueueEvent): |
| 3045 | """ |
| 3046 | Event notifying a setscene task was started |
| 3047 | """ |
| 3048 | def __init__(self, task, stats, rq, noexec=False): |
| 3049 | sceneQueueEvent.__init__(self, task, stats, rq) |
| 3050 | self.noexec = noexec |
| 3051 | |
| 3052 | class runQueueTaskFailed(runQueueEvent): |
| 3053 | """ |
| 3054 | Event notifying a task failed |
| 3055 | """ |
Andrew Geissler | 95ac1b8 | 2021-03-31 14:34:31 -0500 | [diff] [blame] | 3056 | def __init__(self, task, stats, exitcode, rq, fakeroot_log=None): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 3057 | runQueueEvent.__init__(self, task, stats, rq) |
| 3058 | self.exitcode = exitcode |
Andrew Geissler | 95ac1b8 | 2021-03-31 14:34:31 -0500 | [diff] [blame] | 3059 | self.fakeroot_log = fakeroot_log |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 3060 | |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 3061 | def __str__(self): |
Andrew Geissler | 95ac1b8 | 2021-03-31 14:34:31 -0500 | [diff] [blame] | 3062 | if self.fakeroot_log: |
| 3063 | return "Task (%s) failed with exit code '%s' \nPseudo log:\n%s" % (self.taskstring, self.exitcode, self.fakeroot_log) |
| 3064 | else: |
| 3065 | return "Task (%s) failed with exit code '%s'" % (self.taskstring, self.exitcode) |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 3066 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 3067 | class sceneQueueTaskFailed(sceneQueueEvent): |
| 3068 | """ |
| 3069 | Event notifying a setscene task failed |
| 3070 | """ |
| 3071 | def __init__(self, task, stats, exitcode, rq): |
| 3072 | sceneQueueEvent.__init__(self, task, stats, rq) |
| 3073 | self.exitcode = exitcode |
| 3074 | |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 3075 | def __str__(self): |
| 3076 | return "Setscene task (%s) failed with exit code '%s' - real task will be run instead" % (self.taskstring, self.exitcode) |
| 3077 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 3078 | class sceneQueueComplete(sceneQueueEvent): |
| 3079 | """ |
| 3080 | Event when all the sceneQueue tasks are complete |
| 3081 | """ |
| 3082 | def __init__(self, stats, rq): |
| 3083 | self.stats = stats.copy() |
| 3084 | bb.event.Event.__init__(self) |
| 3085 | |
| 3086 | class runQueueTaskCompleted(runQueueEvent): |
| 3087 | """ |
| 3088 | Event notifying a task completed |
| 3089 | """ |
| 3090 | |
| 3091 | class sceneQueueTaskCompleted(sceneQueueEvent): |
| 3092 | """ |
| 3093 | Event notifying a setscene task completed |
| 3094 | """ |
| 3095 | |
| 3096 | class runQueueTaskSkipped(runQueueEvent): |
| 3097 | """ |
| 3098 | Event notifying a task was skipped |
| 3099 | """ |
| 3100 | def __init__(self, task, stats, rq, reason): |
| 3101 | runQueueEvent.__init__(self, task, stats, rq) |
| 3102 | self.reason = reason |
| 3103 | |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 3104 | class taskUniHashUpdate(bb.event.Event): |
| 3105 | """ |
| 3106 | Base runQueue event class |
| 3107 | """ |
| 3108 | def __init__(self, task, unihash): |
| 3109 | self.taskid = task |
| 3110 | self.unihash = unihash |
| 3111 | bb.event.Event.__init__(self) |
| 3112 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 3113 | class runQueuePipe(): |
| 3114 | """ |
| 3115 | Abstraction for a pipe between a worker thread and the server |
| 3116 | """ |
Andrew Geissler | 95ac1b8 | 2021-03-31 14:34:31 -0500 | [diff] [blame] | 3117 | def __init__(self, pipein, pipeout, d, rq, rqexec, fakerootlogs=None): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 3118 | self.input = pipein |
| 3119 | if pipeout: |
| 3120 | pipeout.close() |
| 3121 | bb.utils.nonblockingfd(self.input) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 3122 | self.queue = b"" |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 3123 | self.d = d |
| 3124 | self.rq = rq |
| 3125 | self.rqexec = rqexec |
Andrew Geissler | 95ac1b8 | 2021-03-31 14:34:31 -0500 | [diff] [blame] | 3126 | self.fakerootlogs = fakerootlogs |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 3127 | |
| 3128 | def setrunqueueexec(self, rqexec): |
| 3129 | self.rqexec = rqexec |
| 3130 | |
| 3131 | def read(self): |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 3132 | for workers, name in [(self.rq.worker, "Worker"), (self.rq.fakeworker, "Fakeroot")]: |
| 3133 | for worker in workers.values(): |
| 3134 | worker.process.poll() |
| 3135 | if worker.process.returncode is not None and not self.rq.teardown: |
| 3136 | bb.error("%s process (%s) exited unexpectedly (%s), shutting down..." % (name, worker.process.pid, str(worker.process.returncode))) |
| 3137 | self.rq.finish_runqueue(True) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 3138 | |
| 3139 | start = len(self.queue) |
| 3140 | try: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 3141 | self.queue = self.queue + (self.input.read(102400) or b"") |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 3142 | except (OSError, IOError) as e: |
| 3143 | if e.errno != errno.EAGAIN: |
| 3144 | raise |
| 3145 | end = len(self.queue) |
| 3146 | found = True |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 3147 | while found and self.queue: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 3148 | found = False |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 3149 | index = self.queue.find(b"</event>") |
| 3150 | while index != -1 and self.queue.startswith(b"<event>"): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 3151 | try: |
| 3152 | event = pickle.loads(self.queue[7:index]) |
Andrew Geissler | 475cb72 | 2020-07-10 16:00:51 -0500 | [diff] [blame] | 3153 | except (ValueError, pickle.UnpicklingError, AttributeError, IndexError) as e: |
| 3154 | if isinstance(e, pickle.UnpicklingError) and "truncated" in str(e): |
| 3155 | # The pickled data could contain "</event>" so search for the next occurance |
| 3156 | # unpickling again, this should be the only way an unpickle error could occur |
| 3157 | index = self.queue.find(b"</event>", index + 1) |
| 3158 | continue |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 3159 | bb.msg.fatal("RunQueue", "failed load pickle '%s': '%s'" % (e, self.queue[7:index])) |
| 3160 | bb.event.fire_from_worker(event, self.d) |
Brad Bishop | 08902b0 | 2019-08-20 09:16:51 -0400 | [diff] [blame] | 3161 | if isinstance(event, taskUniHashUpdate): |
| 3162 | self.rqexec.updated_taskhash_queue.append((event.taskid, event.unihash)) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 3163 | found = True |
| 3164 | self.queue = self.queue[index+8:] |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 3165 | index = self.queue.find(b"</event>") |
| 3166 | index = self.queue.find(b"</exitcode>") |
| 3167 | while index != -1 and self.queue.startswith(b"<exitcode>"): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 3168 | try: |
| 3169 | task, status = pickle.loads(self.queue[10:index]) |
Andrew Geissler | 475cb72 | 2020-07-10 16:00:51 -0500 | [diff] [blame] | 3170 | except (ValueError, pickle.UnpicklingError, AttributeError, IndexError) as e: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 3171 | bb.msg.fatal("RunQueue", "failed load pickle '%s': '%s'" % (e, self.queue[10:index])) |
Andrew Geissler | 95ac1b8 | 2021-03-31 14:34:31 -0500 | [diff] [blame] | 3172 | (_, _, _, taskfn) = split_tid_mcfn(task) |
| 3173 | fakerootlog = None |
| 3174 | if self.fakerootlogs and taskfn and taskfn in self.fakerootlogs: |
| 3175 | fakerootlog = self.fakerootlogs[taskfn] |
| 3176 | self.rqexec.runqueue_process_waitpid(task, status, fakerootlog=fakerootlog) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 3177 | found = True |
| 3178 | self.queue = self.queue[index+11:] |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 3179 | index = self.queue.find(b"</exitcode>") |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 3180 | return (end > start) |
| 3181 | |
| 3182 | def close(self): |
| 3183 | while self.read(): |
| 3184 | continue |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 3185 | if self.queue: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 3186 | print("Warning, worker left partial message: %s" % self.queue) |
| 3187 | self.input.close() |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 3188 | |
Andrew Geissler | 7e0e3c0 | 2022-02-25 20:34:39 +0000 | [diff] [blame] | 3189 | def get_setscene_enforce_ignore_tasks(d, targets): |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 3190 | if d.getVar('BB_SETSCENE_ENFORCE') != '1': |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 3191 | return None |
Andrew Geissler | 7e0e3c0 | 2022-02-25 20:34:39 +0000 | [diff] [blame] | 3192 | ignore_tasks = (d.getVar("BB_SETSCENE_ENFORCE_IGNORE_TASKS") or "").split() |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 3193 | outlist = [] |
Andrew Geissler | 7e0e3c0 | 2022-02-25 20:34:39 +0000 | [diff] [blame] | 3194 | for item in ignore_tasks[:]: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 3195 | if item.startswith('%:'): |
Andrew Geissler | c9f7865 | 2020-09-18 14:11:35 -0500 | [diff] [blame] | 3196 | for (mc, target, task, fn) in targets: |
| 3197 | outlist.append(target + ':' + item.split(':')[1]) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 3198 | else: |
| 3199 | outlist.append(item) |
| 3200 | return outlist |
| 3201 | |
Andrew Geissler | 7e0e3c0 | 2022-02-25 20:34:39 +0000 | [diff] [blame] | 3202 | def check_setscene_enforce_ignore_tasks(pn, taskname, ignore_tasks): |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 3203 | import fnmatch |
Andrew Geissler | 7e0e3c0 | 2022-02-25 20:34:39 +0000 | [diff] [blame] | 3204 | if ignore_tasks is not None: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 3205 | item = '%s:%s' % (pn, taskname) |
Andrew Geissler | 7e0e3c0 | 2022-02-25 20:34:39 +0000 | [diff] [blame] | 3206 | for ignore_tasks in ignore_tasks: |
| 3207 | if fnmatch.fnmatch(item, ignore_tasks): |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 3208 | return True |
| 3209 | return False |
| 3210 | return True |