blob: 97cc0fd60fbc2aaeb67c52e29d24b7563d675bb4 [file] [log] [blame]
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001#!/usr/bin/env python3
Brad Bishopc342db32019-05-15 21:57:59 -04002#
3# SPDX-License-Identifier: GPL-2.0-only
4#
Patrick Williamsc124f4f2015-09-15 14:41:29 -05005
6import os
7import sys
8import warnings
9sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
10from bb import fetch2
11import logging
12import bb
13import select
14import errno
15import signal
Patrick Williamsc0f7c042017-02-23 20:41:17 -060016import pickle
Brad Bishop37a0e4d2017-12-04 01:01:44 -050017import traceback
18import queue
Patrick Williamsf1e5d692016-03-30 15:21:19 -050019from multiprocessing import Lock
Brad Bishop37a0e4d2017-12-04 01:01:44 -050020from threading import Thread
Patrick Williamsc124f4f2015-09-15 14:41:29 -050021
Patrick Williamsc0f7c042017-02-23 20:41:17 -060022if sys.getfilesystemencoding() != "utf-8":
Brad Bishopd7bf8c12018-02-25 22:55:05 -050023 sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
Patrick Williamsc0f7c042017-02-23 20:41:17 -060024
Patrick Williamsc124f4f2015-09-15 14:41:29 -050025# Users shouldn't be running this code directly
26if len(sys.argv) != 2 or not sys.argv[1].startswith("decafbad"):
27 print("bitbake-worker is meant for internal execution by bitbake itself, please don't use it standalone.")
28 sys.exit(1)
29
30profiling = False
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050031if sys.argv[1].startswith("decafbadbad"):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050032 profiling = True
33 try:
34 import cProfile as profile
35 except:
36 import profile
37
38# Unbuffer stdout to avoid log truncation in the event
39# of an unorderly exit as well as to provide timely
40# updates to log files for use with tail
41try:
42 if sys.stdout.name == '<stdout>':
Patrick Williamsc0f7c042017-02-23 20:41:17 -060043 import fcntl
44 fl = fcntl.fcntl(sys.stdout.fileno(), fcntl.F_GETFL)
45 fl |= os.O_SYNC
46 fcntl.fcntl(sys.stdout.fileno(), fcntl.F_SETFL, fl)
47 #sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050048except:
49 pass
50
51logger = logging.getLogger("BitBake")
52
Patrick Williamsc124f4f2015-09-15 14:41:29 -050053worker_pipe = sys.stdout.fileno()
54bb.utils.nonblockingfd(worker_pipe)
Patrick Williamsf1e5d692016-03-30 15:21:19 -050055# Need to guard against multiprocessing being used in child processes
56# and multiple processes trying to write to the parent at the same time
57worker_pipe_lock = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -050058
59handler = bb.event.LogHandler()
60logger.addHandler(handler)
61
62if 0:
63 # Code to write out a log file of all events passing through the worker
64 logfilename = "/tmp/workerlogfile"
65 format_str = "%(levelname)s: %(message)s"
66 conlogformat = bb.msg.BBLogFormatter(format_str)
67 consolelog = logging.FileHandler(logfilename)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050068 consolelog.setFormatter(conlogformat)
69 logger.addHandler(consolelog)
70
Brad Bishop37a0e4d2017-12-04 01:01:44 -050071worker_queue = queue.Queue()
Patrick Williamsc124f4f2015-09-15 14:41:29 -050072
73def worker_fire(event, d):
Patrick Williamsc0f7c042017-02-23 20:41:17 -060074 data = b"<event>" + pickle.dumps(event) + b"</event>"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050075 worker_fire_prepickled(data)
76
77def worker_fire_prepickled(event):
78 global worker_queue
79
Brad Bishop37a0e4d2017-12-04 01:01:44 -050080 worker_queue.put(event)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050081
Brad Bishop37a0e4d2017-12-04 01:01:44 -050082#
83# We can end up with write contention with the cooker, it can be trying to send commands
84# and we can be trying to send event data back. Therefore use a separate thread for writing
85# back data to cooker.
86#
87worker_thread_exit = False
Patrick Williamsc124f4f2015-09-15 14:41:29 -050088
Brad Bishop37a0e4d2017-12-04 01:01:44 -050089def worker_flush(worker_queue):
90 worker_queue_int = b""
91 global worker_pipe, worker_thread_exit
Patrick Williamsc124f4f2015-09-15 14:41:29 -050092
Brad Bishop37a0e4d2017-12-04 01:01:44 -050093 while True:
94 try:
95 worker_queue_int = worker_queue_int + worker_queue.get(True, 1)
96 except queue.Empty:
97 pass
98 while (worker_queue_int or not worker_queue.empty()):
99 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500100 (_, ready, _) = select.select([], [worker_pipe], [], 1)
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500101 if not worker_queue.empty():
102 worker_queue_int = worker_queue_int + worker_queue.get()
103 written = os.write(worker_pipe, worker_queue_int)
104 worker_queue_int = worker_queue_int[written:]
105 except (IOError, OSError) as e:
106 if e.errno != errno.EAGAIN and e.errno != errno.EPIPE:
107 raise
108 if worker_thread_exit and worker_queue.empty() and not worker_queue_int:
109 return
110
111worker_thread = Thread(target=worker_flush, args=(worker_queue,))
112worker_thread.start()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500113
114def worker_child_fire(event, d):
115 global worker_pipe
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500116 global worker_pipe_lock
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500117
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600118 data = b"<event>" + pickle.dumps(event) + b"</event>"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500119 try:
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500120 worker_pipe_lock.acquire()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500121 worker_pipe.write(data)
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500122 worker_pipe_lock.release()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500123 except IOError:
124 sigterm_handler(None, None)
125 raise
126
127bb.event.worker_fire = worker_fire
128
129lf = None
130#lf = open("/tmp/workercommandlog", "w+")
131def workerlog_write(msg):
132 if lf:
133 lf.write(msg)
134 lf.flush()
135
136def sigterm_handler(signum, frame):
137 signal.signal(signal.SIGTERM, signal.SIG_DFL)
138 os.killpg(0, signal.SIGTERM)
139 sys.exit()
140
Brad Bishop19323692019-04-05 15:28:33 -0400141def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, taskhash, unihash, appends, taskdepdata, extraconfigdata, quieterrors=False, dry_run_exec=False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500142 # We need to setup the environment BEFORE the fork, since
143 # a fork() or exec*() activates PSEUDO...
144
145 envbackup = {}
146 fakeenv = {}
147 umask = None
148
149 taskdep = workerdata["taskdeps"][fn]
150 if 'umask' in taskdep and taskname in taskdep['umask']:
151 # umask might come in as a number or text string..
152 try:
153 umask = int(taskdep['umask'][taskname],8)
154 except TypeError:
155 umask = taskdep['umask'][taskname]
156
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500157 dry_run = cfg.dry_run or dry_run_exec
158
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500159 # We can't use the fakeroot environment in a dry run as it possibly hasn't been built
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500160 if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not dry_run:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500161 envvars = (workerdata["fakerootenv"][fn] or "").split()
162 for key, value in (var.split('=') for var in envvars):
163 envbackup[key] = os.environ.get(key)
164 os.environ[key] = value
165 fakeenv[key] = value
166
167 fakedirs = (workerdata["fakerootdirs"][fn] or "").split()
168 for p in fakedirs:
169 bb.utils.mkdirhier(p)
170 logger.debug(2, 'Running %s:%s under fakeroot, fakedirs: %s' %
171 (fn, taskname, ', '.join(fakedirs)))
172 else:
173 envvars = (workerdata["fakerootnoenv"][fn] or "").split()
174 for key, value in (var.split('=') for var in envvars):
175 envbackup[key] = os.environ.get(key)
176 os.environ[key] = value
177 fakeenv[key] = value
178
179 sys.stdout.flush()
180 sys.stderr.flush()
181
182 try:
183 pipein, pipeout = os.pipe()
184 pipein = os.fdopen(pipein, 'rb', 4096)
185 pipeout = os.fdopen(pipeout, 'wb', 0)
186 pid = os.fork()
187 except OSError as e:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600188 logger.critical("fork failed: %d (%s)" % (e.errno, e.strerror))
189 sys.exit(1)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500190
191 if pid == 0:
192 def child():
193 global worker_pipe
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500194 global worker_pipe_lock
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500195 pipein.close()
196
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500197 bb.utils.signal_on_parent_exit("SIGTERM")
198
199 # Save out the PID so that the event can include it the
200 # events
201 bb.event.worker_pid = os.getpid()
202 bb.event.worker_fire = worker_child_fire
203 worker_pipe = pipeout
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500204 worker_pipe_lock = Lock()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500205
206 # Make the child the process group leader and ensure no
207 # child process will be controlled by the current terminal
208 # This ensures signals sent to the controlling terminal like Ctrl+C
209 # don't stop the child processes.
210 os.setsid()
Brad Bishop1d80a2e2019-11-15 16:35:03 -0500211
212 signal.signal(signal.SIGTERM, sigterm_handler)
213 # Let SIGHUP exit as SIGTERM
214 signal.signal(signal.SIGHUP, sigterm_handler)
215
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500216 # No stdin
217 newsi = os.open(os.devnull, os.O_RDWR)
218 os.dup2(newsi, sys.stdin.fileno())
219
220 if umask:
221 os.umask(umask)
222
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500223 try:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600224 bb_cache = bb.cache.NoCache(databuilder)
225 (realfn, virtual, mc) = bb.cache.virtualfn2realfn(fn)
226 the_data = databuilder.mcdata[mc]
227 the_data.setVar("BB_WORKERCONTEXT", "1")
228 the_data.setVar("BB_TASKDEPDATA", taskdepdata)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500229 if cfg.limited_deps:
230 the_data.setVar("BB_LIMITEDDEPS", "1")
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600231 the_data.setVar("BUILDNAME", workerdata["buildname"])
232 the_data.setVar("DATE", workerdata["date"])
233 the_data.setVar("TIME", workerdata["time"])
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500234 for varname, value in extraconfigdata.items():
235 the_data.setVar(varname, value)
236
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600237 bb.parse.siggen.set_taskdata(workerdata["sigdata"])
Brad Bishop08902b02019-08-20 09:16:51 -0400238 if "newhashes" in workerdata:
239 bb.parse.siggen.set_taskhashes(workerdata["newhashes"])
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600240 ret = 0
241
242 the_data = bb_cache.loadDataFull(fn, appends)
Brad Bishop19323692019-04-05 15:28:33 -0400243 the_data.setVar('BB_TASKHASH', taskhash)
244 the_data.setVar('BB_UNIHASH', unihash)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500245
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500246 bb.utils.set_process_name("%s:%s" % (the_data.getVar("PN"), taskname.replace("do_", "")))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500247
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500248 # exported_vars() returns a generator which *cannot* be passed to os.environ.update()
249 # successfully. We also need to unset anything from the environment which shouldn't be there
250 exports = bb.data.exported_vars(the_data)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600251
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500252 bb.utils.empty_environment()
253 for e, v in exports:
254 os.environ[e] = v
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600255
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500256 for e in fakeenv:
257 os.environ[e] = fakeenv[e]
258 the_data.setVar(e, fakeenv[e])
259 the_data.setVarFlag(e, 'export', "1")
260
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500261 task_exports = the_data.getVarFlag(taskname, 'exports')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600262 if task_exports:
263 for e in task_exports.split():
264 the_data.setVarFlag(e, 'export', '1')
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500265 v = the_data.getVar(e)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600266 if v is not None:
267 os.environ[e] = v
268
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500269 if quieterrors:
270 the_data.setVarFlag(taskname, "quieterrors", "1")
271
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500272 except Exception:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500273 if not quieterrors:
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500274 logger.critical(traceback.format_exc())
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500275 os._exit(1)
276 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500277 if dry_run:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500278 return 0
279 return bb.build.exec_task(fn, taskname, the_data, cfg.profile)
280 except:
281 os._exit(1)
282 if not profiling:
283 os._exit(child())
284 else:
285 profname = "profile-%s.log" % (fn.replace("/", "-") + "-" + taskname)
286 prof = profile.Profile()
287 try:
288 ret = profile.Profile.runcall(prof, child)
289 finally:
290 prof.dump_stats(profname)
291 bb.utils.process_profilelog(profname)
292 os._exit(ret)
293 else:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600294 for key, value in iter(envbackup.items()):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500295 if value is None:
296 del os.environ[key]
297 else:
298 os.environ[key] = value
299
300 return pid, pipein, pipeout
301
302class runQueueWorkerPipe():
303 """
304 Abstraction for a pipe between a worker thread and the worker server
305 """
306 def __init__(self, pipein, pipeout):
307 self.input = pipein
308 if pipeout:
309 pipeout.close()
310 bb.utils.nonblockingfd(self.input)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600311 self.queue = b""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500312
313 def read(self):
314 start = len(self.queue)
315 try:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600316 self.queue = self.queue + (self.input.read(102400) or b"")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500317 except (OSError, IOError) as e:
318 if e.errno != errno.EAGAIN:
319 raise
320
321 end = len(self.queue)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600322 index = self.queue.find(b"</event>")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500323 while index != -1:
324 worker_fire_prepickled(self.queue[:index+8])
325 self.queue = self.queue[index+8:]
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600326 index = self.queue.find(b"</event>")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500327 return (end > start)
328
329 def close(self):
330 while self.read():
331 continue
332 if len(self.queue) > 0:
333 print("Warning, worker child left partial message: %s" % self.queue)
334 self.input.close()
335
336normalexit = False
337
338class BitbakeWorker(object):
339 def __init__(self, din):
340 self.input = din
341 bb.utils.nonblockingfd(self.input)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600342 self.queue = b""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500343 self.cookercfg = None
344 self.databuilder = None
345 self.data = None
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500346 self.extraconfigdata = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500347 self.build_pids = {}
348 self.build_pipes = {}
349
350 signal.signal(signal.SIGTERM, self.sigterm_exception)
351 # Let SIGHUP exit as SIGTERM
352 signal.signal(signal.SIGHUP, self.sigterm_exception)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500353 if "beef" in sys.argv[1]:
354 bb.utils.set_process_name("Worker (Fakeroot)")
355 else:
356 bb.utils.set_process_name("Worker")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500357
358 def sigterm_exception(self, signum, stackframe):
359 if signum == signal.SIGTERM:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500360 bb.warn("Worker received SIGTERM, shutting down...")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500361 elif signum == signal.SIGHUP:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500362 bb.warn("Worker received SIGHUP, shutting down...")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500363 self.handle_finishnow(None)
364 signal.signal(signal.SIGTERM, signal.SIG_DFL)
365 os.kill(os.getpid(), signal.SIGTERM)
366
367 def serve(self):
368 while True:
369 (ready, _, _) = select.select([self.input] + [i.input for i in self.build_pipes.values()], [] , [], 1)
370 if self.input in ready:
371 try:
372 r = self.input.read()
373 if len(r) == 0:
374 # EOF on pipe, server must have terminated
375 self.sigterm_exception(signal.SIGTERM, None)
376 self.queue = self.queue + r
377 except (OSError, IOError):
378 pass
379 if len(self.queue):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600380 self.handle_item(b"cookerconfig", self.handle_cookercfg)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500381 self.handle_item(b"extraconfigdata", self.handle_extraconfigdata)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600382 self.handle_item(b"workerdata", self.handle_workerdata)
Brad Bishop08902b02019-08-20 09:16:51 -0400383 self.handle_item(b"newtaskhashes", self.handle_newtaskhashes)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600384 self.handle_item(b"runtask", self.handle_runtask)
385 self.handle_item(b"finishnow", self.handle_finishnow)
386 self.handle_item(b"ping", self.handle_ping)
387 self.handle_item(b"quit", self.handle_quit)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500388
389 for pipe in self.build_pipes:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500390 if self.build_pipes[pipe].input in ready:
391 self.build_pipes[pipe].read()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500392 if len(self.build_pids):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500393 while self.process_waitpid():
394 continue
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500395
396
397 def handle_item(self, item, func):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600398 if self.queue.startswith(b"<" + item + b">"):
399 index = self.queue.find(b"</" + item + b">")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500400 while index != -1:
401 func(self.queue[(len(item) + 2):index])
402 self.queue = self.queue[(index + len(item) + 3):]
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600403 index = self.queue.find(b"</" + item + b">")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500404
405 def handle_cookercfg(self, data):
406 self.cookercfg = pickle.loads(data)
407 self.databuilder = bb.cookerdata.CookerDataBuilder(self.cookercfg, worker=True)
408 self.databuilder.parseBaseConfiguration()
409 self.data = self.databuilder.data
410
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500411 def handle_extraconfigdata(self, data):
412 self.extraconfigdata = pickle.loads(data)
413
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500414 def handle_workerdata(self, data):
415 self.workerdata = pickle.loads(data)
Andrew Geissler82c905d2020-04-13 13:39:40 -0500416 bb.msg.loggerDefaultLogLevel = self.workerdata["logdefaultlevel"]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500417 bb.msg.loggerDefaultVerbose = self.workerdata["logdefaultverbose"]
418 bb.msg.loggerVerboseLogs = self.workerdata["logdefaultverboselogs"]
419 bb.msg.loggerDefaultDomains = self.workerdata["logdefaultdomain"]
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600420 for mc in self.databuilder.mcdata:
421 self.databuilder.mcdata[mc].setVar("PRSERV_HOST", self.workerdata["prhost"])
Brad Bishopa34c0302019-09-23 22:34:48 -0400422 self.databuilder.mcdata[mc].setVar("BB_HASHSERVE", self.workerdata["hashservaddr"])
Brad Bishop08902b02019-08-20 09:16:51 -0400423
424 def handle_newtaskhashes(self, data):
425 self.workerdata["newhashes"] = pickle.loads(data)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500426
427 def handle_ping(self, _):
428 workerlog_write("Handling ping\n")
429
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600430 logger.warning("Pong from bitbake-worker!")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500431
432 def handle_quit(self, data):
433 workerlog_write("Handling quit\n")
434
435 global normalexit
436 normalexit = True
437 sys.exit(0)
438
439 def handle_runtask(self, data):
Brad Bishop19323692019-04-05 15:28:33 -0400440 fn, task, taskname, taskhash, unihash, quieterrors, appends, taskdepdata, dry_run_exec = pickle.loads(data)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500441 workerlog_write("Handling runtask %s %s %s\n" % (task, fn, taskname))
442
Brad Bishop19323692019-04-05 15:28:33 -0400443 pid, pipein, pipeout = fork_off_task(self.cookercfg, self.data, self.databuilder, self.workerdata, fn, task, taskname, taskhash, unihash, appends, taskdepdata, self.extraconfigdata, quieterrors, dry_run_exec)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500444
445 self.build_pids[pid] = task
446 self.build_pipes[pid] = runQueueWorkerPipe(pipein, pipeout)
447
448 def process_waitpid(self):
449 """
450 Return none is there are no processes awaiting result collection, otherwise
451 collect the process exit codes and close the information pipe.
452 """
453 try:
454 pid, status = os.waitpid(-1, os.WNOHANG)
455 if pid == 0 or os.WIFSTOPPED(status):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500456 return False
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500457 except OSError:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500458 return False
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500459
460 workerlog_write("Exit code of %s for pid %s\n" % (status, pid))
461
462 if os.WIFEXITED(status):
463 status = os.WEXITSTATUS(status)
464 elif os.WIFSIGNALED(status):
465 # Per shell conventions for $?, when a process exits due to
466 # a signal, we return an exit code of 128 + SIGNUM
467 status = 128 + os.WTERMSIG(status)
468
469 task = self.build_pids[pid]
470 del self.build_pids[pid]
471
472 self.build_pipes[pid].close()
473 del self.build_pipes[pid]
474
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600475 worker_fire_prepickled(b"<exitcode>" + pickle.dumps((task, status)) + b"</exitcode>")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500476
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500477 return True
478
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500479 def handle_finishnow(self, _):
480 if self.build_pids:
481 logger.info("Sending SIGTERM to remaining %s tasks", len(self.build_pids))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600482 for k, v in iter(self.build_pids.items()):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500483 try:
484 os.kill(-k, signal.SIGTERM)
485 os.waitpid(-1, 0)
486 except:
487 pass
488 for pipe in self.build_pipes:
489 self.build_pipes[pipe].read()
490
491try:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600492 worker = BitbakeWorker(os.fdopen(sys.stdin.fileno(), 'rb'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500493 if not profiling:
494 worker.serve()
495 else:
496 profname = "profile-worker.log"
497 prof = profile.Profile()
498 try:
499 profile.Profile.runcall(prof, worker.serve)
500 finally:
501 prof.dump_stats(profname)
502 bb.utils.process_profilelog(profname)
503except BaseException as e:
504 if not normalexit:
505 import traceback
506 sys.stderr.write(traceback.format_exc())
507 sys.stderr.write(str(e))
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500508
509worker_thread_exit = True
510worker_thread.join()
511
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500512workerlog_write("exitting")
513sys.exit(0)