blob: 6776cadda3dad768cbd8ce916e86c02497f09be9 [file] [log] [blame]
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001#!/usr/bin/env python3
Brad Bishopc342db32019-05-15 21:57:59 -04002#
3# SPDX-License-Identifier: GPL-2.0-only
4#
Patrick Williamsc124f4f2015-09-15 14:41:29 -05005
6import os
7import sys
8import warnings
9sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
10from bb import fetch2
11import logging
12import bb
13import select
14import errno
15import signal
Patrick Williamsc0f7c042017-02-23 20:41:17 -060016import pickle
Brad Bishop37a0e4d2017-12-04 01:01:44 -050017import traceback
18import queue
Patrick Williamsf1e5d692016-03-30 15:21:19 -050019from multiprocessing import Lock
Brad Bishop37a0e4d2017-12-04 01:01:44 -050020from threading import Thread
Patrick Williamsc124f4f2015-09-15 14:41:29 -050021
Patrick Williamsc0f7c042017-02-23 20:41:17 -060022if sys.getfilesystemencoding() != "utf-8":
Brad Bishopd7bf8c12018-02-25 22:55:05 -050023 sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
Patrick Williamsc0f7c042017-02-23 20:41:17 -060024
Patrick Williamsc124f4f2015-09-15 14:41:29 -050025# Users shouldn't be running this code directly
26if len(sys.argv) != 2 or not sys.argv[1].startswith("decafbad"):
27 print("bitbake-worker is meant for internal execution by bitbake itself, please don't use it standalone.")
28 sys.exit(1)
29
30profiling = False
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050031if sys.argv[1].startswith("decafbadbad"):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050032 profiling = True
33 try:
34 import cProfile as profile
35 except:
36 import profile
37
38# Unbuffer stdout to avoid log truncation in the event
39# of an unorderly exit as well as to provide timely
40# updates to log files for use with tail
41try:
42 if sys.stdout.name == '<stdout>':
Patrick Williamsc0f7c042017-02-23 20:41:17 -060043 import fcntl
44 fl = fcntl.fcntl(sys.stdout.fileno(), fcntl.F_GETFL)
45 fl |= os.O_SYNC
46 fcntl.fcntl(sys.stdout.fileno(), fcntl.F_SETFL, fl)
47 #sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050048except:
49 pass
50
51logger = logging.getLogger("BitBake")
52
Patrick Williamsc124f4f2015-09-15 14:41:29 -050053worker_pipe = sys.stdout.fileno()
54bb.utils.nonblockingfd(worker_pipe)
Patrick Williamsf1e5d692016-03-30 15:21:19 -050055# Need to guard against multiprocessing being used in child processes
56# and multiple processes trying to write to the parent at the same time
57worker_pipe_lock = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -050058
59handler = bb.event.LogHandler()
60logger.addHandler(handler)
61
62if 0:
63 # Code to write out a log file of all events passing through the worker
64 logfilename = "/tmp/workerlogfile"
65 format_str = "%(levelname)s: %(message)s"
66 conlogformat = bb.msg.BBLogFormatter(format_str)
67 consolelog = logging.FileHandler(logfilename)
68 bb.msg.addDefaultlogFilter(consolelog)
69 consolelog.setFormatter(conlogformat)
70 logger.addHandler(consolelog)
71
Brad Bishop37a0e4d2017-12-04 01:01:44 -050072worker_queue = queue.Queue()
Patrick Williamsc124f4f2015-09-15 14:41:29 -050073
74def worker_fire(event, d):
Patrick Williamsc0f7c042017-02-23 20:41:17 -060075 data = b"<event>" + pickle.dumps(event) + b"</event>"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050076 worker_fire_prepickled(data)
77
78def worker_fire_prepickled(event):
79 global worker_queue
80
Brad Bishop37a0e4d2017-12-04 01:01:44 -050081 worker_queue.put(event)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050082
Brad Bishop37a0e4d2017-12-04 01:01:44 -050083#
84# We can end up with write contention with the cooker, it can be trying to send commands
85# and we can be trying to send event data back. Therefore use a separate thread for writing
86# back data to cooker.
87#
88worker_thread_exit = False
Patrick Williamsc124f4f2015-09-15 14:41:29 -050089
Brad Bishop37a0e4d2017-12-04 01:01:44 -050090def worker_flush(worker_queue):
91 worker_queue_int = b""
92 global worker_pipe, worker_thread_exit
Patrick Williamsc124f4f2015-09-15 14:41:29 -050093
Brad Bishop37a0e4d2017-12-04 01:01:44 -050094 while True:
95 try:
96 worker_queue_int = worker_queue_int + worker_queue.get(True, 1)
97 except queue.Empty:
98 pass
99 while (worker_queue_int or not worker_queue.empty()):
100 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500101 (_, ready, _) = select.select([], [worker_pipe], [], 1)
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500102 if not worker_queue.empty():
103 worker_queue_int = worker_queue_int + worker_queue.get()
104 written = os.write(worker_pipe, worker_queue_int)
105 worker_queue_int = worker_queue_int[written:]
106 except (IOError, OSError) as e:
107 if e.errno != errno.EAGAIN and e.errno != errno.EPIPE:
108 raise
109 if worker_thread_exit and worker_queue.empty() and not worker_queue_int:
110 return
111
112worker_thread = Thread(target=worker_flush, args=(worker_queue,))
113worker_thread.start()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500114
115def worker_child_fire(event, d):
116 global worker_pipe
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500117 global worker_pipe_lock
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500118
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600119 data = b"<event>" + pickle.dumps(event) + b"</event>"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500120 try:
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500121 worker_pipe_lock.acquire()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500122 worker_pipe.write(data)
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500123 worker_pipe_lock.release()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500124 except IOError:
125 sigterm_handler(None, None)
126 raise
127
128bb.event.worker_fire = worker_fire
129
130lf = None
131#lf = open("/tmp/workercommandlog", "w+")
132def workerlog_write(msg):
133 if lf:
134 lf.write(msg)
135 lf.flush()
136
137def sigterm_handler(signum, frame):
138 signal.signal(signal.SIGTERM, signal.SIG_DFL)
139 os.killpg(0, signal.SIGTERM)
140 sys.exit()
141
Brad Bishop19323692019-04-05 15:28:33 -0400142def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, taskhash, unihash, appends, taskdepdata, extraconfigdata, quieterrors=False, dry_run_exec=False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500143 # We need to setup the environment BEFORE the fork, since
144 # a fork() or exec*() activates PSEUDO...
145
146 envbackup = {}
147 fakeenv = {}
148 umask = None
149
150 taskdep = workerdata["taskdeps"][fn]
151 if 'umask' in taskdep and taskname in taskdep['umask']:
152 # umask might come in as a number or text string..
153 try:
154 umask = int(taskdep['umask'][taskname],8)
155 except TypeError:
156 umask = taskdep['umask'][taskname]
157
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500158 dry_run = cfg.dry_run or dry_run_exec
159
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500160 # We can't use the fakeroot environment in a dry run as it possibly hasn't been built
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500161 if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not dry_run:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500162 envvars = (workerdata["fakerootenv"][fn] or "").split()
163 for key, value in (var.split('=') for var in envvars):
164 envbackup[key] = os.environ.get(key)
165 os.environ[key] = value
166 fakeenv[key] = value
167
168 fakedirs = (workerdata["fakerootdirs"][fn] or "").split()
169 for p in fakedirs:
170 bb.utils.mkdirhier(p)
171 logger.debug(2, 'Running %s:%s under fakeroot, fakedirs: %s' %
172 (fn, taskname, ', '.join(fakedirs)))
173 else:
174 envvars = (workerdata["fakerootnoenv"][fn] or "").split()
175 for key, value in (var.split('=') for var in envvars):
176 envbackup[key] = os.environ.get(key)
177 os.environ[key] = value
178 fakeenv[key] = value
179
180 sys.stdout.flush()
181 sys.stderr.flush()
182
183 try:
184 pipein, pipeout = os.pipe()
185 pipein = os.fdopen(pipein, 'rb', 4096)
186 pipeout = os.fdopen(pipeout, 'wb', 0)
187 pid = os.fork()
188 except OSError as e:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600189 logger.critical("fork failed: %d (%s)" % (e.errno, e.strerror))
190 sys.exit(1)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500191
192 if pid == 0:
193 def child():
194 global worker_pipe
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500195 global worker_pipe_lock
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500196 pipein.close()
197
198 signal.signal(signal.SIGTERM, sigterm_handler)
199 # Let SIGHUP exit as SIGTERM
200 signal.signal(signal.SIGHUP, sigterm_handler)
201 bb.utils.signal_on_parent_exit("SIGTERM")
202
203 # Save out the PID so that the event can include it the
204 # events
205 bb.event.worker_pid = os.getpid()
206 bb.event.worker_fire = worker_child_fire
207 worker_pipe = pipeout
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500208 worker_pipe_lock = Lock()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500209
210 # Make the child the process group leader and ensure no
211 # child process will be controlled by the current terminal
212 # This ensures signals sent to the controlling terminal like Ctrl+C
213 # don't stop the child processes.
214 os.setsid()
215 # No stdin
216 newsi = os.open(os.devnull, os.O_RDWR)
217 os.dup2(newsi, sys.stdin.fileno())
218
219 if umask:
220 os.umask(umask)
221
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500222 try:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600223 bb_cache = bb.cache.NoCache(databuilder)
224 (realfn, virtual, mc) = bb.cache.virtualfn2realfn(fn)
225 the_data = databuilder.mcdata[mc]
226 the_data.setVar("BB_WORKERCONTEXT", "1")
227 the_data.setVar("BB_TASKDEPDATA", taskdepdata)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500228 if cfg.limited_deps:
229 the_data.setVar("BB_LIMITEDDEPS", "1")
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600230 the_data.setVar("BUILDNAME", workerdata["buildname"])
231 the_data.setVar("DATE", workerdata["date"])
232 the_data.setVar("TIME", workerdata["time"])
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500233 for varname, value in extraconfigdata.items():
234 the_data.setVar(varname, value)
235
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600236 bb.parse.siggen.set_taskdata(workerdata["sigdata"])
Brad Bishop08902b02019-08-20 09:16:51 -0400237 if "newhashes" in workerdata:
238 bb.parse.siggen.set_taskhashes(workerdata["newhashes"])
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600239 ret = 0
240
241 the_data = bb_cache.loadDataFull(fn, appends)
Brad Bishop19323692019-04-05 15:28:33 -0400242 the_data.setVar('BB_TASKHASH', taskhash)
243 the_data.setVar('BB_UNIHASH', unihash)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500244
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500245 bb.utils.set_process_name("%s:%s" % (the_data.getVar("PN"), taskname.replace("do_", "")))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500246
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500247 # exported_vars() returns a generator which *cannot* be passed to os.environ.update()
248 # successfully. We also need to unset anything from the environment which shouldn't be there
249 exports = bb.data.exported_vars(the_data)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600250
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500251 bb.utils.empty_environment()
252 for e, v in exports:
253 os.environ[e] = v
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600254
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500255 for e in fakeenv:
256 os.environ[e] = fakeenv[e]
257 the_data.setVar(e, fakeenv[e])
258 the_data.setVarFlag(e, 'export', "1")
259
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500260 task_exports = the_data.getVarFlag(taskname, 'exports')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600261 if task_exports:
262 for e in task_exports.split():
263 the_data.setVarFlag(e, 'export', '1')
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500264 v = the_data.getVar(e)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600265 if v is not None:
266 os.environ[e] = v
267
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500268 if quieterrors:
269 the_data.setVarFlag(taskname, "quieterrors", "1")
270
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500271 except Exception:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500272 if not quieterrors:
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500273 logger.critical(traceback.format_exc())
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500274 os._exit(1)
275 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500276 if dry_run:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500277 return 0
278 return bb.build.exec_task(fn, taskname, the_data, cfg.profile)
279 except:
280 os._exit(1)
281 if not profiling:
282 os._exit(child())
283 else:
284 profname = "profile-%s.log" % (fn.replace("/", "-") + "-" + taskname)
285 prof = profile.Profile()
286 try:
287 ret = profile.Profile.runcall(prof, child)
288 finally:
289 prof.dump_stats(profname)
290 bb.utils.process_profilelog(profname)
291 os._exit(ret)
292 else:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600293 for key, value in iter(envbackup.items()):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500294 if value is None:
295 del os.environ[key]
296 else:
297 os.environ[key] = value
298
299 return pid, pipein, pipeout
300
301class runQueueWorkerPipe():
302 """
303 Abstraction for a pipe between a worker thread and the worker server
304 """
305 def __init__(self, pipein, pipeout):
306 self.input = pipein
307 if pipeout:
308 pipeout.close()
309 bb.utils.nonblockingfd(self.input)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600310 self.queue = b""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500311
312 def read(self):
313 start = len(self.queue)
314 try:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600315 self.queue = self.queue + (self.input.read(102400) or b"")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500316 except (OSError, IOError) as e:
317 if e.errno != errno.EAGAIN:
318 raise
319
320 end = len(self.queue)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600321 index = self.queue.find(b"</event>")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500322 while index != -1:
323 worker_fire_prepickled(self.queue[:index+8])
324 self.queue = self.queue[index+8:]
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600325 index = self.queue.find(b"</event>")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500326 return (end > start)
327
328 def close(self):
329 while self.read():
330 continue
331 if len(self.queue) > 0:
332 print("Warning, worker child left partial message: %s" % self.queue)
333 self.input.close()
334
335normalexit = False
336
337class BitbakeWorker(object):
338 def __init__(self, din):
339 self.input = din
340 bb.utils.nonblockingfd(self.input)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600341 self.queue = b""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500342 self.cookercfg = None
343 self.databuilder = None
344 self.data = None
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500345 self.extraconfigdata = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500346 self.build_pids = {}
347 self.build_pipes = {}
348
349 signal.signal(signal.SIGTERM, self.sigterm_exception)
350 # Let SIGHUP exit as SIGTERM
351 signal.signal(signal.SIGHUP, self.sigterm_exception)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500352 if "beef" in sys.argv[1]:
353 bb.utils.set_process_name("Worker (Fakeroot)")
354 else:
355 bb.utils.set_process_name("Worker")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500356
357 def sigterm_exception(self, signum, stackframe):
358 if signum == signal.SIGTERM:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500359 bb.warn("Worker received SIGTERM, shutting down...")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500360 elif signum == signal.SIGHUP:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500361 bb.warn("Worker received SIGHUP, shutting down...")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500362 self.handle_finishnow(None)
363 signal.signal(signal.SIGTERM, signal.SIG_DFL)
364 os.kill(os.getpid(), signal.SIGTERM)
365
366 def serve(self):
367 while True:
368 (ready, _, _) = select.select([self.input] + [i.input for i in self.build_pipes.values()], [] , [], 1)
369 if self.input in ready:
370 try:
371 r = self.input.read()
372 if len(r) == 0:
373 # EOF on pipe, server must have terminated
374 self.sigterm_exception(signal.SIGTERM, None)
375 self.queue = self.queue + r
376 except (OSError, IOError):
377 pass
378 if len(self.queue):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600379 self.handle_item(b"cookerconfig", self.handle_cookercfg)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500380 self.handle_item(b"extraconfigdata", self.handle_extraconfigdata)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600381 self.handle_item(b"workerdata", self.handle_workerdata)
Brad Bishop08902b02019-08-20 09:16:51 -0400382 self.handle_item(b"newtaskhashes", self.handle_newtaskhashes)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600383 self.handle_item(b"runtask", self.handle_runtask)
384 self.handle_item(b"finishnow", self.handle_finishnow)
385 self.handle_item(b"ping", self.handle_ping)
386 self.handle_item(b"quit", self.handle_quit)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500387
388 for pipe in self.build_pipes:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500389 if self.build_pipes[pipe].input in ready:
390 self.build_pipes[pipe].read()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500391 if len(self.build_pids):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500392 while self.process_waitpid():
393 continue
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500394
395
396 def handle_item(self, item, func):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600397 if self.queue.startswith(b"<" + item + b">"):
398 index = self.queue.find(b"</" + item + b">")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500399 while index != -1:
400 func(self.queue[(len(item) + 2):index])
401 self.queue = self.queue[(index + len(item) + 3):]
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600402 index = self.queue.find(b"</" + item + b">")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500403
404 def handle_cookercfg(self, data):
405 self.cookercfg = pickle.loads(data)
406 self.databuilder = bb.cookerdata.CookerDataBuilder(self.cookercfg, worker=True)
407 self.databuilder.parseBaseConfiguration()
408 self.data = self.databuilder.data
409
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500410 def handle_extraconfigdata(self, data):
411 self.extraconfigdata = pickle.loads(data)
412
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500413 def handle_workerdata(self, data):
414 self.workerdata = pickle.loads(data)
415 bb.msg.loggerDefaultDebugLevel = self.workerdata["logdefaultdebug"]
416 bb.msg.loggerDefaultVerbose = self.workerdata["logdefaultverbose"]
417 bb.msg.loggerVerboseLogs = self.workerdata["logdefaultverboselogs"]
418 bb.msg.loggerDefaultDomains = self.workerdata["logdefaultdomain"]
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600419 for mc in self.databuilder.mcdata:
420 self.databuilder.mcdata[mc].setVar("PRSERV_HOST", self.workerdata["prhost"])
Brad Bishopa34c0302019-09-23 22:34:48 -0400421 self.databuilder.mcdata[mc].setVar("BB_HASHSERVE", self.workerdata["hashservaddr"])
Brad Bishop08902b02019-08-20 09:16:51 -0400422
423 def handle_newtaskhashes(self, data):
424 self.workerdata["newhashes"] = pickle.loads(data)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500425
426 def handle_ping(self, _):
427 workerlog_write("Handling ping\n")
428
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600429 logger.warning("Pong from bitbake-worker!")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500430
431 def handle_quit(self, data):
432 workerlog_write("Handling quit\n")
433
434 global normalexit
435 normalexit = True
436 sys.exit(0)
437
438 def handle_runtask(self, data):
Brad Bishop19323692019-04-05 15:28:33 -0400439 fn, task, taskname, taskhash, unihash, quieterrors, appends, taskdepdata, dry_run_exec = pickle.loads(data)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500440 workerlog_write("Handling runtask %s %s %s\n" % (task, fn, taskname))
441
Brad Bishop19323692019-04-05 15:28:33 -0400442 pid, pipein, pipeout = fork_off_task(self.cookercfg, self.data, self.databuilder, self.workerdata, fn, task, taskname, taskhash, unihash, appends, taskdepdata, self.extraconfigdata, quieterrors, dry_run_exec)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500443
444 self.build_pids[pid] = task
445 self.build_pipes[pid] = runQueueWorkerPipe(pipein, pipeout)
446
447 def process_waitpid(self):
448 """
449 Return none is there are no processes awaiting result collection, otherwise
450 collect the process exit codes and close the information pipe.
451 """
452 try:
453 pid, status = os.waitpid(-1, os.WNOHANG)
454 if pid == 0 or os.WIFSTOPPED(status):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500455 return False
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500456 except OSError:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500457 return False
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500458
459 workerlog_write("Exit code of %s for pid %s\n" % (status, pid))
460
461 if os.WIFEXITED(status):
462 status = os.WEXITSTATUS(status)
463 elif os.WIFSIGNALED(status):
464 # Per shell conventions for $?, when a process exits due to
465 # a signal, we return an exit code of 128 + SIGNUM
466 status = 128 + os.WTERMSIG(status)
467
468 task = self.build_pids[pid]
469 del self.build_pids[pid]
470
471 self.build_pipes[pid].close()
472 del self.build_pipes[pid]
473
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600474 worker_fire_prepickled(b"<exitcode>" + pickle.dumps((task, status)) + b"</exitcode>")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500475
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500476 return True
477
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500478 def handle_finishnow(self, _):
479 if self.build_pids:
480 logger.info("Sending SIGTERM to remaining %s tasks", len(self.build_pids))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600481 for k, v in iter(self.build_pids.items()):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500482 try:
483 os.kill(-k, signal.SIGTERM)
484 os.waitpid(-1, 0)
485 except:
486 pass
487 for pipe in self.build_pipes:
488 self.build_pipes[pipe].read()
489
490try:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600491 worker = BitbakeWorker(os.fdopen(sys.stdin.fileno(), 'rb'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500492 if not profiling:
493 worker.serve()
494 else:
495 profname = "profile-worker.log"
496 prof = profile.Profile()
497 try:
498 profile.Profile.runcall(prof, worker.serve)
499 finally:
500 prof.dump_stats(profname)
501 bb.utils.process_profilelog(profname)
502except BaseException as e:
503 if not normalexit:
504 import traceback
505 sys.stderr.write(traceback.format_exc())
506 sys.stderr.write(str(e))
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500507
508worker_thread_exit = True
509worker_thread.join()
510
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500511workerlog_write("exitting")
512sys.exit(0)