blob: 7765b9368b02a3f7cefc2ff76eeca466c9183d9a [file] [log] [blame]
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001#!/usr/bin/env python3
Brad Bishopc342db32019-05-15 21:57:59 -04002#
3# SPDX-License-Identifier: GPL-2.0-only
4#
Patrick Williamsc124f4f2015-09-15 14:41:29 -05005
6import os
7import sys
8import warnings
9sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
10from bb import fetch2
11import logging
12import bb
13import select
14import errno
15import signal
Patrick Williamsc0f7c042017-02-23 20:41:17 -060016import pickle
Brad Bishop37a0e4d2017-12-04 01:01:44 -050017import traceback
18import queue
Patrick Williamsf1e5d692016-03-30 15:21:19 -050019from multiprocessing import Lock
Brad Bishop37a0e4d2017-12-04 01:01:44 -050020from threading import Thread
Patrick Williamsc124f4f2015-09-15 14:41:29 -050021
Patrick Williamsc0f7c042017-02-23 20:41:17 -060022if sys.getfilesystemencoding() != "utf-8":
Brad Bishopd7bf8c12018-02-25 22:55:05 -050023 sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
Patrick Williamsc0f7c042017-02-23 20:41:17 -060024
Patrick Williamsc124f4f2015-09-15 14:41:29 -050025# Users shouldn't be running this code directly
26if len(sys.argv) != 2 or not sys.argv[1].startswith("decafbad"):
27 print("bitbake-worker is meant for internal execution by bitbake itself, please don't use it standalone.")
28 sys.exit(1)
29
30profiling = False
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050031if sys.argv[1].startswith("decafbadbad"):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050032 profiling = True
33 try:
34 import cProfile as profile
35 except:
36 import profile
37
38# Unbuffer stdout to avoid log truncation in the event
39# of an unorderly exit as well as to provide timely
40# updates to log files for use with tail
41try:
42 if sys.stdout.name == '<stdout>':
Patrick Williamsc0f7c042017-02-23 20:41:17 -060043 import fcntl
44 fl = fcntl.fcntl(sys.stdout.fileno(), fcntl.F_GETFL)
45 fl |= os.O_SYNC
46 fcntl.fcntl(sys.stdout.fileno(), fcntl.F_SETFL, fl)
47 #sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050048except:
49 pass
50
51logger = logging.getLogger("BitBake")
52
Patrick Williamsc124f4f2015-09-15 14:41:29 -050053worker_pipe = sys.stdout.fileno()
54bb.utils.nonblockingfd(worker_pipe)
Patrick Williamsf1e5d692016-03-30 15:21:19 -050055# Need to guard against multiprocessing being used in child processes
56# and multiple processes trying to write to the parent at the same time
57worker_pipe_lock = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -050058
59handler = bb.event.LogHandler()
60logger.addHandler(handler)
61
62if 0:
63 # Code to write out a log file of all events passing through the worker
64 logfilename = "/tmp/workerlogfile"
65 format_str = "%(levelname)s: %(message)s"
66 conlogformat = bb.msg.BBLogFormatter(format_str)
67 consolelog = logging.FileHandler(logfilename)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050068 consolelog.setFormatter(conlogformat)
69 logger.addHandler(consolelog)
70
Brad Bishop37a0e4d2017-12-04 01:01:44 -050071worker_queue = queue.Queue()
Patrick Williamsc124f4f2015-09-15 14:41:29 -050072
73def worker_fire(event, d):
Patrick Williamsc0f7c042017-02-23 20:41:17 -060074 data = b"<event>" + pickle.dumps(event) + b"</event>"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050075 worker_fire_prepickled(data)
76
77def worker_fire_prepickled(event):
78 global worker_queue
79
Brad Bishop37a0e4d2017-12-04 01:01:44 -050080 worker_queue.put(event)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050081
Brad Bishop37a0e4d2017-12-04 01:01:44 -050082#
83# We can end up with write contention with the cooker, it can be trying to send commands
84# and we can be trying to send event data back. Therefore use a separate thread for writing
85# back data to cooker.
86#
87worker_thread_exit = False
Patrick Williamsc124f4f2015-09-15 14:41:29 -050088
Brad Bishop37a0e4d2017-12-04 01:01:44 -050089def worker_flush(worker_queue):
90 worker_queue_int = b""
91 global worker_pipe, worker_thread_exit
Patrick Williamsc124f4f2015-09-15 14:41:29 -050092
Brad Bishop37a0e4d2017-12-04 01:01:44 -050093 while True:
94 try:
95 worker_queue_int = worker_queue_int + worker_queue.get(True, 1)
96 except queue.Empty:
97 pass
98 while (worker_queue_int or not worker_queue.empty()):
99 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500100 (_, ready, _) = select.select([], [worker_pipe], [], 1)
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500101 if not worker_queue.empty():
102 worker_queue_int = worker_queue_int + worker_queue.get()
103 written = os.write(worker_pipe, worker_queue_int)
104 worker_queue_int = worker_queue_int[written:]
105 except (IOError, OSError) as e:
106 if e.errno != errno.EAGAIN and e.errno != errno.EPIPE:
107 raise
108 if worker_thread_exit and worker_queue.empty() and not worker_queue_int:
109 return
110
111worker_thread = Thread(target=worker_flush, args=(worker_queue,))
112worker_thread.start()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500113
114def worker_child_fire(event, d):
115 global worker_pipe
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500116 global worker_pipe_lock
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500117
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600118 data = b"<event>" + pickle.dumps(event) + b"</event>"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500119 try:
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500120 worker_pipe_lock.acquire()
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600121 while(len(data)):
122 written = worker_pipe.write(data)
123 data = data[written:]
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500124 worker_pipe_lock.release()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500125 except IOError:
126 sigterm_handler(None, None)
127 raise
128
129bb.event.worker_fire = worker_fire
130
131lf = None
132#lf = open("/tmp/workercommandlog", "w+")
133def workerlog_write(msg):
134 if lf:
135 lf.write(msg)
136 lf.flush()
137
138def sigterm_handler(signum, frame):
139 signal.signal(signal.SIGTERM, signal.SIG_DFL)
140 os.killpg(0, signal.SIGTERM)
141 sys.exit()
142
Brad Bishop19323692019-04-05 15:28:33 -0400143def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, taskhash, unihash, appends, taskdepdata, extraconfigdata, quieterrors=False, dry_run_exec=False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500144 # We need to setup the environment BEFORE the fork, since
145 # a fork() or exec*() activates PSEUDO...
146
147 envbackup = {}
148 fakeenv = {}
149 umask = None
150
151 taskdep = workerdata["taskdeps"][fn]
152 if 'umask' in taskdep and taskname in taskdep['umask']:
Andrew Geissler9b4d8b02021-02-19 12:26:16 -0600153 umask = taskdep['umask'][taskname]
154 elif workerdata["umask"]:
155 umask = workerdata["umask"]
156 if umask:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500157 # umask might come in as a number or text string..
158 try:
Andrew Geissler9b4d8b02021-02-19 12:26:16 -0600159 umask = int(umask, 8)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500160 except TypeError:
Andrew Geissler9b4d8b02021-02-19 12:26:16 -0600161 pass
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500162
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500163 dry_run = cfg.dry_run or dry_run_exec
164
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500165 # We can't use the fakeroot environment in a dry run as it possibly hasn't been built
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500166 if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not dry_run:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500167 envvars = (workerdata["fakerootenv"][fn] or "").split()
168 for key, value in (var.split('=') for var in envvars):
169 envbackup[key] = os.environ.get(key)
170 os.environ[key] = value
171 fakeenv[key] = value
172
173 fakedirs = (workerdata["fakerootdirs"][fn] or "").split()
174 for p in fakedirs:
175 bb.utils.mkdirhier(p)
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600176 logger.debug2('Running %s:%s under fakeroot, fakedirs: %s' %
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500177 (fn, taskname, ', '.join(fakedirs)))
178 else:
179 envvars = (workerdata["fakerootnoenv"][fn] or "").split()
180 for key, value in (var.split('=') for var in envvars):
181 envbackup[key] = os.environ.get(key)
182 os.environ[key] = value
183 fakeenv[key] = value
184
185 sys.stdout.flush()
186 sys.stderr.flush()
187
188 try:
189 pipein, pipeout = os.pipe()
190 pipein = os.fdopen(pipein, 'rb', 4096)
191 pipeout = os.fdopen(pipeout, 'wb', 0)
192 pid = os.fork()
193 except OSError as e:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600194 logger.critical("fork failed: %d (%s)" % (e.errno, e.strerror))
195 sys.exit(1)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500196
197 if pid == 0:
198 def child():
199 global worker_pipe
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500200 global worker_pipe_lock
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500201 pipein.close()
202
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500203 bb.utils.signal_on_parent_exit("SIGTERM")
204
205 # Save out the PID so that the event can include it the
206 # events
207 bb.event.worker_pid = os.getpid()
208 bb.event.worker_fire = worker_child_fire
209 worker_pipe = pipeout
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500210 worker_pipe_lock = Lock()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500211
212 # Make the child the process group leader and ensure no
213 # child process will be controlled by the current terminal
214 # This ensures signals sent to the controlling terminal like Ctrl+C
215 # don't stop the child processes.
216 os.setsid()
Brad Bishop1d80a2e2019-11-15 16:35:03 -0500217
218 signal.signal(signal.SIGTERM, sigterm_handler)
219 # Let SIGHUP exit as SIGTERM
220 signal.signal(signal.SIGHUP, sigterm_handler)
221
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500222 # No stdin
223 newsi = os.open(os.devnull, os.O_RDWR)
224 os.dup2(newsi, sys.stdin.fileno())
225
226 if umask:
227 os.umask(umask)
228
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500229 try:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600230 bb_cache = bb.cache.NoCache(databuilder)
231 (realfn, virtual, mc) = bb.cache.virtualfn2realfn(fn)
232 the_data = databuilder.mcdata[mc]
233 the_data.setVar("BB_WORKERCONTEXT", "1")
234 the_data.setVar("BB_TASKDEPDATA", taskdepdata)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500235 if cfg.limited_deps:
236 the_data.setVar("BB_LIMITEDDEPS", "1")
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600237 the_data.setVar("BUILDNAME", workerdata["buildname"])
238 the_data.setVar("DATE", workerdata["date"])
239 the_data.setVar("TIME", workerdata["time"])
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500240 for varname, value in extraconfigdata.items():
241 the_data.setVar(varname, value)
242
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600243 bb.parse.siggen.set_taskdata(workerdata["sigdata"])
Brad Bishop08902b02019-08-20 09:16:51 -0400244 if "newhashes" in workerdata:
245 bb.parse.siggen.set_taskhashes(workerdata["newhashes"])
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600246 ret = 0
247
248 the_data = bb_cache.loadDataFull(fn, appends)
Brad Bishop19323692019-04-05 15:28:33 -0400249 the_data.setVar('BB_TASKHASH', taskhash)
250 the_data.setVar('BB_UNIHASH', unihash)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500251
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500252 bb.utils.set_process_name("%s:%s" % (the_data.getVar("PN"), taskname.replace("do_", "")))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500253
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500254 # exported_vars() returns a generator which *cannot* be passed to os.environ.update()
255 # successfully. We also need to unset anything from the environment which shouldn't be there
256 exports = bb.data.exported_vars(the_data)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600257
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500258 bb.utils.empty_environment()
259 for e, v in exports:
260 os.environ[e] = v
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600261
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500262 for e in fakeenv:
263 os.environ[e] = fakeenv[e]
264 the_data.setVar(e, fakeenv[e])
265 the_data.setVarFlag(e, 'export', "1")
266
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500267 task_exports = the_data.getVarFlag(taskname, 'exports')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600268 if task_exports:
269 for e in task_exports.split():
270 the_data.setVarFlag(e, 'export', '1')
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500271 v = the_data.getVar(e)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600272 if v is not None:
273 os.environ[e] = v
274
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500275 if quieterrors:
276 the_data.setVarFlag(taskname, "quieterrors", "1")
277
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500278 except Exception:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500279 if not quieterrors:
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500280 logger.critical(traceback.format_exc())
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500281 os._exit(1)
282 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500283 if dry_run:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500284 return 0
285 return bb.build.exec_task(fn, taskname, the_data, cfg.profile)
286 except:
287 os._exit(1)
288 if not profiling:
289 os._exit(child())
290 else:
291 profname = "profile-%s.log" % (fn.replace("/", "-") + "-" + taskname)
292 prof = profile.Profile()
293 try:
294 ret = profile.Profile.runcall(prof, child)
295 finally:
296 prof.dump_stats(profname)
297 bb.utils.process_profilelog(profname)
298 os._exit(ret)
299 else:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600300 for key, value in iter(envbackup.items()):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500301 if value is None:
302 del os.environ[key]
303 else:
304 os.environ[key] = value
305
306 return pid, pipein, pipeout
307
308class runQueueWorkerPipe():
309 """
310 Abstraction for a pipe between a worker thread and the worker server
311 """
312 def __init__(self, pipein, pipeout):
313 self.input = pipein
314 if pipeout:
315 pipeout.close()
316 bb.utils.nonblockingfd(self.input)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600317 self.queue = b""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500318
319 def read(self):
320 start = len(self.queue)
321 try:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600322 self.queue = self.queue + (self.input.read(102400) or b"")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500323 except (OSError, IOError) as e:
324 if e.errno != errno.EAGAIN:
325 raise
326
327 end = len(self.queue)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600328 index = self.queue.find(b"</event>")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500329 while index != -1:
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600330 msg = self.queue[:index+8]
331 assert msg.startswith(b"<event>") and msg.count(b"<event>") == 1
332 worker_fire_prepickled(msg)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500333 self.queue = self.queue[index+8:]
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600334 index = self.queue.find(b"</event>")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500335 return (end > start)
336
337 def close(self):
338 while self.read():
339 continue
340 if len(self.queue) > 0:
341 print("Warning, worker child left partial message: %s" % self.queue)
342 self.input.close()
343
344normalexit = False
345
346class BitbakeWorker(object):
347 def __init__(self, din):
348 self.input = din
349 bb.utils.nonblockingfd(self.input)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600350 self.queue = b""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500351 self.cookercfg = None
352 self.databuilder = None
353 self.data = None
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500354 self.extraconfigdata = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500355 self.build_pids = {}
356 self.build_pipes = {}
357
358 signal.signal(signal.SIGTERM, self.sigterm_exception)
359 # Let SIGHUP exit as SIGTERM
360 signal.signal(signal.SIGHUP, self.sigterm_exception)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500361 if "beef" in sys.argv[1]:
362 bb.utils.set_process_name("Worker (Fakeroot)")
363 else:
364 bb.utils.set_process_name("Worker")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500365
366 def sigterm_exception(self, signum, stackframe):
367 if signum == signal.SIGTERM:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500368 bb.warn("Worker received SIGTERM, shutting down...")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500369 elif signum == signal.SIGHUP:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500370 bb.warn("Worker received SIGHUP, shutting down...")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500371 self.handle_finishnow(None)
372 signal.signal(signal.SIGTERM, signal.SIG_DFL)
373 os.kill(os.getpid(), signal.SIGTERM)
374
375 def serve(self):
376 while True:
377 (ready, _, _) = select.select([self.input] + [i.input for i in self.build_pipes.values()], [] , [], 1)
378 if self.input in ready:
379 try:
380 r = self.input.read()
381 if len(r) == 0:
382 # EOF on pipe, server must have terminated
383 self.sigterm_exception(signal.SIGTERM, None)
384 self.queue = self.queue + r
385 except (OSError, IOError):
386 pass
387 if len(self.queue):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600388 self.handle_item(b"cookerconfig", self.handle_cookercfg)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500389 self.handle_item(b"extraconfigdata", self.handle_extraconfigdata)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600390 self.handle_item(b"workerdata", self.handle_workerdata)
Brad Bishop08902b02019-08-20 09:16:51 -0400391 self.handle_item(b"newtaskhashes", self.handle_newtaskhashes)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600392 self.handle_item(b"runtask", self.handle_runtask)
393 self.handle_item(b"finishnow", self.handle_finishnow)
394 self.handle_item(b"ping", self.handle_ping)
395 self.handle_item(b"quit", self.handle_quit)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500396
397 for pipe in self.build_pipes:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500398 if self.build_pipes[pipe].input in ready:
399 self.build_pipes[pipe].read()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500400 if len(self.build_pids):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500401 while self.process_waitpid():
402 continue
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500403
404
405 def handle_item(self, item, func):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600406 if self.queue.startswith(b"<" + item + b">"):
407 index = self.queue.find(b"</" + item + b">")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500408 while index != -1:
409 func(self.queue[(len(item) + 2):index])
410 self.queue = self.queue[(index + len(item) + 3):]
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600411 index = self.queue.find(b"</" + item + b">")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500412
413 def handle_cookercfg(self, data):
414 self.cookercfg = pickle.loads(data)
415 self.databuilder = bb.cookerdata.CookerDataBuilder(self.cookercfg, worker=True)
416 self.databuilder.parseBaseConfiguration()
417 self.data = self.databuilder.data
418
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500419 def handle_extraconfigdata(self, data):
420 self.extraconfigdata = pickle.loads(data)
421
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500422 def handle_workerdata(self, data):
423 self.workerdata = pickle.loads(data)
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500424 bb.build.verboseShellLogging = self.workerdata["build_verbose_shell"]
425 bb.build.verboseStdoutLogging = self.workerdata["build_verbose_stdout"]
Andrew Geissler82c905d2020-04-13 13:39:40 -0500426 bb.msg.loggerDefaultLogLevel = self.workerdata["logdefaultlevel"]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500427 bb.msg.loggerDefaultDomains = self.workerdata["logdefaultdomain"]
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600428 for mc in self.databuilder.mcdata:
429 self.databuilder.mcdata[mc].setVar("PRSERV_HOST", self.workerdata["prhost"])
Brad Bishopa34c0302019-09-23 22:34:48 -0400430 self.databuilder.mcdata[mc].setVar("BB_HASHSERVE", self.workerdata["hashservaddr"])
Brad Bishop08902b02019-08-20 09:16:51 -0400431
432 def handle_newtaskhashes(self, data):
433 self.workerdata["newhashes"] = pickle.loads(data)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500434
435 def handle_ping(self, _):
436 workerlog_write("Handling ping\n")
437
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600438 logger.warning("Pong from bitbake-worker!")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500439
440 def handle_quit(self, data):
441 workerlog_write("Handling quit\n")
442
443 global normalexit
444 normalexit = True
445 sys.exit(0)
446
447 def handle_runtask(self, data):
Brad Bishop19323692019-04-05 15:28:33 -0400448 fn, task, taskname, taskhash, unihash, quieterrors, appends, taskdepdata, dry_run_exec = pickle.loads(data)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500449 workerlog_write("Handling runtask %s %s %s\n" % (task, fn, taskname))
450
Brad Bishop19323692019-04-05 15:28:33 -0400451 pid, pipein, pipeout = fork_off_task(self.cookercfg, self.data, self.databuilder, self.workerdata, fn, task, taskname, taskhash, unihash, appends, taskdepdata, self.extraconfigdata, quieterrors, dry_run_exec)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500452
453 self.build_pids[pid] = task
454 self.build_pipes[pid] = runQueueWorkerPipe(pipein, pipeout)
455
456 def process_waitpid(self):
457 """
458 Return none is there are no processes awaiting result collection, otherwise
459 collect the process exit codes and close the information pipe.
460 """
461 try:
462 pid, status = os.waitpid(-1, os.WNOHANG)
463 if pid == 0 or os.WIFSTOPPED(status):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500464 return False
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500465 except OSError:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500466 return False
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500467
468 workerlog_write("Exit code of %s for pid %s\n" % (status, pid))
469
470 if os.WIFEXITED(status):
471 status = os.WEXITSTATUS(status)
472 elif os.WIFSIGNALED(status):
473 # Per shell conventions for $?, when a process exits due to
474 # a signal, we return an exit code of 128 + SIGNUM
475 status = 128 + os.WTERMSIG(status)
476
477 task = self.build_pids[pid]
478 del self.build_pids[pid]
479
480 self.build_pipes[pid].close()
481 del self.build_pipes[pid]
482
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600483 worker_fire_prepickled(b"<exitcode>" + pickle.dumps((task, status)) + b"</exitcode>")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500484
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500485 return True
486
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500487 def handle_finishnow(self, _):
488 if self.build_pids:
489 logger.info("Sending SIGTERM to remaining %s tasks", len(self.build_pids))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600490 for k, v in iter(self.build_pids.items()):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500491 try:
492 os.kill(-k, signal.SIGTERM)
493 os.waitpid(-1, 0)
494 except:
495 pass
496 for pipe in self.build_pipes:
497 self.build_pipes[pipe].read()
498
499try:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600500 worker = BitbakeWorker(os.fdopen(sys.stdin.fileno(), 'rb'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500501 if not profiling:
502 worker.serve()
503 else:
504 profname = "profile-worker.log"
505 prof = profile.Profile()
506 try:
507 profile.Profile.runcall(prof, worker.serve)
508 finally:
509 prof.dump_stats(profname)
510 bb.utils.process_profilelog(profname)
511except BaseException as e:
512 if not normalexit:
513 import traceback
514 sys.stderr.write(traceback.format_exc())
515 sys.stderr.write(str(e))
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500516
517worker_thread_exit = True
518worker_thread.join()
519
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500520workerlog_write("exitting")
521sys.exit(0)