blob: 9518be77fbef80043f8b995bb4173f9a08297f4b [file] [log] [blame]
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001"""
2BitBake progress handling code
3"""
4
5# Copyright (C) 2016 Intel Corporation
6#
Brad Bishopc342db32019-05-15 21:57:59 -04007# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc0f7c042017-02-23 20:41:17 -06008#
Patrick Williamsc0f7c042017-02-23 20:41:17 -06009
Patrick Williamsc0f7c042017-02-23 20:41:17 -060010import re
11import time
12import inspect
13import bb.event
14import bb.build
Brad Bishop15ae2502019-06-18 21:44:24 -040015from bb.build import StdoutNoopContextManager
Patrick Williamsc0f7c042017-02-23 20:41:17 -060016
Andrew Geissler635e0e42020-08-21 15:58:33 -050017
18# from https://stackoverflow.com/a/14693789/221061
19ANSI_ESCAPE_REGEX = re.compile(r'\x1B\[[0-?]*[ -/]*[@-~]')
20
21
22def filter_color(string):
23 """
24 Filter ANSI escape codes out of |string|, return new string
25 """
26 return ANSI_ESCAPE_REGEX.sub('', string)
27
28
29def filter_color_n(string):
30 """
31 Filter ANSI escape codes out of |string|, returns tuple of
32 (new string, # of ANSI codes removed)
33 """
34 return ANSI_ESCAPE_REGEX.subn('', string)
35
36
37class ProgressHandler:
Patrick Williamsc0f7c042017-02-23 20:41:17 -060038 """
39 Base class that can pretend to be a file object well enough to be
40 used to build objects to intercept console output and determine the
41 progress of some operation.
42 """
43 def __init__(self, d, outfile=None):
44 self._progress = 0
45 self._data = d
46 self._lastevent = 0
47 if outfile:
48 self._outfile = outfile
49 else:
Brad Bishop15ae2502019-06-18 21:44:24 -040050 self._outfile = StdoutNoopContextManager()
51
52 def __enter__(self):
53 self._outfile.__enter__()
54 return self
55
56 def __exit__(self, *excinfo):
57 self._outfile.__exit__(*excinfo)
Patrick Williamsc0f7c042017-02-23 20:41:17 -060058
59 def _fire_progress(self, taskprogress, rate=None):
60 """Internal function to fire the progress event"""
61 bb.event.fire(bb.build.TaskProgress(taskprogress, rate), self._data)
62
63 def write(self, string):
64 self._outfile.write(string)
65
66 def flush(self):
67 self._outfile.flush()
68
69 def update(self, progress, rate=None):
70 ts = time.time()
71 if progress > 100:
72 progress = 100
73 if progress != self._progress or self._lastevent + 1 < ts:
74 self._fire_progress(progress, rate)
75 self._lastevent = ts
76 self._progress = progress
77
Andrew Geissler635e0e42020-08-21 15:58:33 -050078
Patrick Williamsc0f7c042017-02-23 20:41:17 -060079class LineFilterProgressHandler(ProgressHandler):
80 """
81 A ProgressHandler variant that provides the ability to filter out
82 the lines if they contain progress information. Additionally, it
83 filters out anything before the last line feed on a line. This can
84 be used to keep the logs clean of output that we've only enabled for
85 getting progress, assuming that that can be done on a per-line
86 basis.
87 """
88 def __init__(self, d, outfile=None):
89 self._linebuffer = ''
Andrew Geissler635e0e42020-08-21 15:58:33 -050090 super().__init__(d, outfile)
Patrick Williamsc0f7c042017-02-23 20:41:17 -060091
92 def write(self, string):
93 self._linebuffer += string
94 while True:
95 breakpos = self._linebuffer.find('\n') + 1
96 if breakpos == 0:
Andrew Geisslerc926e172021-05-07 16:11:35 -050097 # for the case when the line with progress ends with only '\r'
98 breakpos = self._linebuffer.find('\r') + 1
99 if breakpos == 0:
100 break
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600101 line = self._linebuffer[:breakpos]
102 self._linebuffer = self._linebuffer[breakpos:]
103 # Drop any line feeds and anything that precedes them
104 lbreakpos = line.rfind('\r') + 1
Andrew Geisslerc926e172021-05-07 16:11:35 -0500105 if lbreakpos and lbreakpos != breakpos:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600106 line = line[lbreakpos:]
Andrew Geissler635e0e42020-08-21 15:58:33 -0500107 if self.writeline(filter_color(line)):
108 super().write(line)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600109
110 def writeline(self, line):
111 return True
112
Andrew Geissler635e0e42020-08-21 15:58:33 -0500113
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600114class BasicProgressHandler(ProgressHandler):
115 def __init__(self, d, regex=r'(\d+)%', outfile=None):
Andrew Geissler635e0e42020-08-21 15:58:33 -0500116 super().__init__(d, outfile)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600117 self._regex = re.compile(regex)
118 # Send an initial progress event so the bar gets shown
119 self._fire_progress(0)
120
121 def write(self, string):
Andrew Geissler635e0e42020-08-21 15:58:33 -0500122 percs = self._regex.findall(filter_color(string))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600123 if percs:
124 progress = int(percs[-1])
125 self.update(progress)
Andrew Geissler635e0e42020-08-21 15:58:33 -0500126 super().write(string)
127
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600128
129class OutOfProgressHandler(ProgressHandler):
130 def __init__(self, d, regex, outfile=None):
Andrew Geissler635e0e42020-08-21 15:58:33 -0500131 super().__init__(d, outfile)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600132 self._regex = re.compile(regex)
133 # Send an initial progress event so the bar gets shown
134 self._fire_progress(0)
135
136 def write(self, string):
Andrew Geissler635e0e42020-08-21 15:58:33 -0500137 nums = self._regex.findall(filter_color(string))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600138 if nums:
139 progress = (float(nums[-1][0]) / float(nums[-1][1])) * 100
140 self.update(progress)
Andrew Geissler635e0e42020-08-21 15:58:33 -0500141 super().write(string)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600142
Andrew Geissler635e0e42020-08-21 15:58:33 -0500143
144class MultiStageProgressReporter:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600145 """
146 Class which allows reporting progress without the caller
147 having to know where they are in the overall sequence. Useful
148 for tasks made up of python code spread across multiple
149 classes / functions - the progress reporter object can
150 be passed around or stored at the object level and calls
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000151 to next_stage() and update() made wherever needed.
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600152 """
153 def __init__(self, d, stage_weights, debug=False):
154 """
155 Initialise the progress reporter.
156
157 Parameters:
158 * d: the datastore (needed for firing the events)
159 * stage_weights: a list of weight values, one for each stage.
160 The value is scaled internally so you only need to specify
161 values relative to other values in the list, so if there
162 are two stages and the first takes 2s and the second takes
163 10s you would specify [2, 10] (or [1, 5], it doesn't matter).
164 * debug: specify True (and ensure you call finish() at the end)
165 in order to show a printout of the calculated stage weights
166 based on timing each stage. Use this to determine what the
167 weights should be when you're not sure.
168 """
169 self._data = d
170 total = sum(stage_weights)
171 self._stage_weights = [float(x)/total for x in stage_weights]
172 self._stage = -1
173 self._base_progress = 0
174 # Send an initial progress event so the bar gets shown
175 self._fire_progress(0)
176 self._debug = debug
177 self._finished = False
178 if self._debug:
179 self._last_time = time.time()
180 self._stage_times = []
181 self._stage_total = None
182 self._callers = []
183
Brad Bishop15ae2502019-06-18 21:44:24 -0400184 def __enter__(self):
185 return self
186
187 def __exit__(self, *excinfo):
188 pass
189
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600190 def _fire_progress(self, taskprogress):
191 bb.event.fire(bb.build.TaskProgress(taskprogress), self._data)
192
193 def next_stage(self, stage_total=None):
194 """
195 Move to the next stage.
196 Parameters:
197 * stage_total: optional total for progress within the stage,
198 see update() for details
199 NOTE: you need to call this before the first stage.
200 """
201 self._stage += 1
202 self._stage_total = stage_total
203 if self._stage == 0:
204 # First stage
205 if self._debug:
206 self._last_time = time.time()
207 else:
208 if self._stage < len(self._stage_weights):
209 self._base_progress = sum(self._stage_weights[:self._stage]) * 100
210 if self._debug:
211 currtime = time.time()
212 self._stage_times.append(currtime - self._last_time)
213 self._last_time = currtime
214 self._callers.append(inspect.getouterframes(inspect.currentframe())[1])
215 elif not self._debug:
216 bb.warn('ProgressReporter: current stage beyond declared number of stages')
217 self._base_progress = 100
218 self._fire_progress(self._base_progress)
219
220 def update(self, stage_progress):
221 """
222 Update progress within the current stage.
223 Parameters:
224 * stage_progress: progress value within the stage. If stage_total
225 was specified when next_stage() was last called, then this
226 value is considered to be out of stage_total, otherwise it should
227 be a percentage value from 0 to 100.
228 """
Andrew Geissler635e0e42020-08-21 15:58:33 -0500229 progress = None
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600230 if self._stage_total:
231 stage_progress = (float(stage_progress) / self._stage_total) * 100
232 if self._stage < 0:
233 bb.warn('ProgressReporter: update called before first call to next_stage()')
234 elif self._stage < len(self._stage_weights):
235 progress = self._base_progress + (stage_progress * self._stage_weights[self._stage])
236 else:
237 progress = self._base_progress
Andrew Geissler635e0e42020-08-21 15:58:33 -0500238 if progress:
239 if progress > 100:
240 progress = 100
241 self._fire_progress(progress)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600242
243 def finish(self):
244 if self._finished:
245 return
246 self._finished = True
247 if self._debug:
248 import math
249 self._stage_times.append(time.time() - self._last_time)
250 mintime = max(min(self._stage_times), 0.01)
251 self._callers.append(None)
252 stage_weights = [int(math.ceil(x / mintime)) for x in self._stage_times]
253 bb.warn('Stage weights: %s' % stage_weights)
254 out = []
255 for stage_weight, caller in zip(stage_weights, self._callers):
256 if caller:
257 out.append('Up to %s:%d: %d' % (caller[1], caller[2], stage_weight))
258 else:
259 out.append('Up to finish: %d' % stage_weight)
260 bb.warn('Stage times:\n %s' % '\n '.join(out))
261
Andrew Geissler635e0e42020-08-21 15:58:33 -0500262
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600263class MultiStageProcessProgressReporter(MultiStageProgressReporter):
264 """
265 Version of MultiStageProgressReporter intended for use with
266 standalone processes (such as preparing the runqueue)
267 """
268 def __init__(self, d, processname, stage_weights, debug=False):
269 self._processname = processname
270 self._started = False
Andrew Geissler635e0e42020-08-21 15:58:33 -0500271 super().__init__(d, stage_weights, debug)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600272
273 def start(self):
274 if not self._started:
275 bb.event.fire(bb.event.ProcessStarted(self._processname, 100), self._data)
276 self._started = True
277
278 def _fire_progress(self, taskprogress):
279 if taskprogress == 0:
280 self.start()
281 return
282 bb.event.fire(bb.event.ProcessProgress(self._processname, taskprogress), self._data)
283
284 def finish(self):
285 MultiStageProgressReporter.finish(self)
286 bb.event.fire(bb.event.ProcessFinished(self._processname), self._data)
287
Andrew Geissler635e0e42020-08-21 15:58:33 -0500288
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600289class DummyMultiStageProcessProgressReporter(MultiStageProgressReporter):
290 """
291 MultiStageProcessProgressReporter that takes the calls and does nothing
292 with them (to avoid a bunch of "if progress_reporter:" checks)
293 """
294 def __init__(self):
Andrew Geissler635e0e42020-08-21 15:58:33 -0500295 super().__init__(None, [])
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600296
297 def _fire_progress(self, taskprogress, rate=None):
298 pass
299
300 def start(self):
301 pass
302
303 def next_stage(self, stage_total=None):
304 pass
305
306 def update(self, stage_progress):
307 pass
308
309 def finish(self):
310 pass