blob: 3aaf3c2444b88c2b63aeba899622d8a369350c87 [file] [log] [blame]
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001#!/usr/bin/env python3
Brad Bishopc342db32019-05-15 21:57:59 -04002#
3# SPDX-License-Identifier: GPL-2.0-only
4#
Patrick Williamsc124f4f2015-09-15 14:41:29 -05005
6import os
7import sys
8import warnings
Andrew Geissler5199d832021-09-24 16:47:35 -05009warnings.simplefilter("default")
Patrick Williamsc124f4f2015-09-15 14:41:29 -050010sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
11from bb import fetch2
12import logging
13import bb
14import select
15import errno
16import signal
Patrick Williamsc0f7c042017-02-23 20:41:17 -060017import pickle
Brad Bishop37a0e4d2017-12-04 01:01:44 -050018import traceback
19import queue
Patrick Williams93c203f2021-10-06 16:15:23 -050020import shlex
21import subprocess
Patrick Williamsf1e5d692016-03-30 15:21:19 -050022from multiprocessing import Lock
Brad Bishop37a0e4d2017-12-04 01:01:44 -050023from threading import Thread
Patrick Williamsc124f4f2015-09-15 14:41:29 -050024
Patrick Williamsc0f7c042017-02-23 20:41:17 -060025if sys.getfilesystemencoding() != "utf-8":
Brad Bishopd7bf8c12018-02-25 22:55:05 -050026 sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
Patrick Williamsc0f7c042017-02-23 20:41:17 -060027
Patrick Williamsc124f4f2015-09-15 14:41:29 -050028# Users shouldn't be running this code directly
29if len(sys.argv) != 2 or not sys.argv[1].startswith("decafbad"):
30 print("bitbake-worker is meant for internal execution by bitbake itself, please don't use it standalone.")
31 sys.exit(1)
32
33profiling = False
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050034if sys.argv[1].startswith("decafbadbad"):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050035 profiling = True
36 try:
37 import cProfile as profile
38 except:
39 import profile
40
41# Unbuffer stdout to avoid log truncation in the event
42# of an unorderly exit as well as to provide timely
43# updates to log files for use with tail
44try:
45 if sys.stdout.name == '<stdout>':
Patrick Williamsc0f7c042017-02-23 20:41:17 -060046 import fcntl
47 fl = fcntl.fcntl(sys.stdout.fileno(), fcntl.F_GETFL)
48 fl |= os.O_SYNC
49 fcntl.fcntl(sys.stdout.fileno(), fcntl.F_SETFL, fl)
50 #sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050051except:
52 pass
53
54logger = logging.getLogger("BitBake")
55
Patrick Williamsc124f4f2015-09-15 14:41:29 -050056worker_pipe = sys.stdout.fileno()
57bb.utils.nonblockingfd(worker_pipe)
Patrick Williamsf1e5d692016-03-30 15:21:19 -050058# Need to guard against multiprocessing being used in child processes
59# and multiple processes trying to write to the parent at the same time
60worker_pipe_lock = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -050061
62handler = bb.event.LogHandler()
63logger.addHandler(handler)
64
65if 0:
66 # Code to write out a log file of all events passing through the worker
67 logfilename = "/tmp/workerlogfile"
68 format_str = "%(levelname)s: %(message)s"
69 conlogformat = bb.msg.BBLogFormatter(format_str)
70 consolelog = logging.FileHandler(logfilename)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050071 consolelog.setFormatter(conlogformat)
72 logger.addHandler(consolelog)
73
Brad Bishop37a0e4d2017-12-04 01:01:44 -050074worker_queue = queue.Queue()
Patrick Williamsc124f4f2015-09-15 14:41:29 -050075
76def worker_fire(event, d):
Patrick Williamsc0f7c042017-02-23 20:41:17 -060077 data = b"<event>" + pickle.dumps(event) + b"</event>"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050078 worker_fire_prepickled(data)
79
80def worker_fire_prepickled(event):
81 global worker_queue
82
Brad Bishop37a0e4d2017-12-04 01:01:44 -050083 worker_queue.put(event)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050084
Brad Bishop37a0e4d2017-12-04 01:01:44 -050085#
86# We can end up with write contention with the cooker, it can be trying to send commands
87# and we can be trying to send event data back. Therefore use a separate thread for writing
88# back data to cooker.
89#
90worker_thread_exit = False
Patrick Williamsc124f4f2015-09-15 14:41:29 -050091
Brad Bishop37a0e4d2017-12-04 01:01:44 -050092def worker_flush(worker_queue):
93 worker_queue_int = b""
94 global worker_pipe, worker_thread_exit
Patrick Williamsc124f4f2015-09-15 14:41:29 -050095
Brad Bishop37a0e4d2017-12-04 01:01:44 -050096 while True:
97 try:
98 worker_queue_int = worker_queue_int + worker_queue.get(True, 1)
99 except queue.Empty:
100 pass
101 while (worker_queue_int or not worker_queue.empty()):
102 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500103 (_, ready, _) = select.select([], [worker_pipe], [], 1)
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500104 if not worker_queue.empty():
105 worker_queue_int = worker_queue_int + worker_queue.get()
106 written = os.write(worker_pipe, worker_queue_int)
107 worker_queue_int = worker_queue_int[written:]
108 except (IOError, OSError) as e:
109 if e.errno != errno.EAGAIN and e.errno != errno.EPIPE:
110 raise
111 if worker_thread_exit and worker_queue.empty() and not worker_queue_int:
112 return
113
114worker_thread = Thread(target=worker_flush, args=(worker_queue,))
115worker_thread.start()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500116
117def worker_child_fire(event, d):
118 global worker_pipe
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500119 global worker_pipe_lock
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500120
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600121 data = b"<event>" + pickle.dumps(event) + b"</event>"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500122 try:
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500123 worker_pipe_lock.acquire()
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600124 while(len(data)):
125 written = worker_pipe.write(data)
126 data = data[written:]
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500127 worker_pipe_lock.release()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500128 except IOError:
129 sigterm_handler(None, None)
130 raise
131
132bb.event.worker_fire = worker_fire
133
134lf = None
135#lf = open("/tmp/workercommandlog", "w+")
136def workerlog_write(msg):
137 if lf:
138 lf.write(msg)
139 lf.flush()
140
141def sigterm_handler(signum, frame):
142 signal.signal(signal.SIGTERM, signal.SIG_DFL)
143 os.killpg(0, signal.SIGTERM)
144 sys.exit()
145
Brad Bishop19323692019-04-05 15:28:33 -0400146def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, taskhash, unihash, appends, taskdepdata, extraconfigdata, quieterrors=False, dry_run_exec=False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500147 # We need to setup the environment BEFORE the fork, since
148 # a fork() or exec*() activates PSEUDO...
149
150 envbackup = {}
Patrick Williams93c203f2021-10-06 16:15:23 -0500151 fakeroot = False
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500152 fakeenv = {}
153 umask = None
154
Andrew Geissler595f6302022-01-24 19:11:47 +0000155 uid = os.getuid()
156 gid = os.getgid()
157
158
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500159 taskdep = workerdata["taskdeps"][fn]
160 if 'umask' in taskdep and taskname in taskdep['umask']:
Andrew Geissler9b4d8b02021-02-19 12:26:16 -0600161 umask = taskdep['umask'][taskname]
162 elif workerdata["umask"]:
163 umask = workerdata["umask"]
164 if umask:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500165 # umask might come in as a number or text string..
166 try:
Andrew Geissler9b4d8b02021-02-19 12:26:16 -0600167 umask = int(umask, 8)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500168 except TypeError:
Andrew Geissler9b4d8b02021-02-19 12:26:16 -0600169 pass
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500170
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500171 dry_run = cfg.dry_run or dry_run_exec
172
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500173 # We can't use the fakeroot environment in a dry run as it possibly hasn't been built
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500174 if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not dry_run:
Patrick Williams93c203f2021-10-06 16:15:23 -0500175 fakeroot = True
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500176 envvars = (workerdata["fakerootenv"][fn] or "").split()
177 for key, value in (var.split('=') for var in envvars):
178 envbackup[key] = os.environ.get(key)
179 os.environ[key] = value
180 fakeenv[key] = value
181
182 fakedirs = (workerdata["fakerootdirs"][fn] or "").split()
183 for p in fakedirs:
184 bb.utils.mkdirhier(p)
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600185 logger.debug2('Running %s:%s under fakeroot, fakedirs: %s' %
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500186 (fn, taskname, ', '.join(fakedirs)))
187 else:
188 envvars = (workerdata["fakerootnoenv"][fn] or "").split()
189 for key, value in (var.split('=') for var in envvars):
190 envbackup[key] = os.environ.get(key)
191 os.environ[key] = value
192 fakeenv[key] = value
193
194 sys.stdout.flush()
195 sys.stderr.flush()
196
197 try:
198 pipein, pipeout = os.pipe()
199 pipein = os.fdopen(pipein, 'rb', 4096)
200 pipeout = os.fdopen(pipeout, 'wb', 0)
201 pid = os.fork()
202 except OSError as e:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600203 logger.critical("fork failed: %d (%s)" % (e.errno, e.strerror))
204 sys.exit(1)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500205
206 if pid == 0:
207 def child():
208 global worker_pipe
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500209 global worker_pipe_lock
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500210 pipein.close()
211
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500212 bb.utils.signal_on_parent_exit("SIGTERM")
213
214 # Save out the PID so that the event can include it the
215 # events
216 bb.event.worker_pid = os.getpid()
217 bb.event.worker_fire = worker_child_fire
218 worker_pipe = pipeout
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500219 worker_pipe_lock = Lock()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500220
221 # Make the child the process group leader and ensure no
222 # child process will be controlled by the current terminal
223 # This ensures signals sent to the controlling terminal like Ctrl+C
224 # don't stop the child processes.
225 os.setsid()
Brad Bishop1d80a2e2019-11-15 16:35:03 -0500226
227 signal.signal(signal.SIGTERM, sigterm_handler)
228 # Let SIGHUP exit as SIGTERM
229 signal.signal(signal.SIGHUP, sigterm_handler)
230
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500231 # No stdin
232 newsi = os.open(os.devnull, os.O_RDWR)
233 os.dup2(newsi, sys.stdin.fileno())
234
235 if umask:
236 os.umask(umask)
237
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500238 try:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600239 bb_cache = bb.cache.NoCache(databuilder)
240 (realfn, virtual, mc) = bb.cache.virtualfn2realfn(fn)
241 the_data = databuilder.mcdata[mc]
242 the_data.setVar("BB_WORKERCONTEXT", "1")
243 the_data.setVar("BB_TASKDEPDATA", taskdepdata)
Andrew Geisslereff27472021-10-29 15:35:00 -0500244 the_data.setVar('BB_CURRENTTASK', taskname.replace("do_", ""))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500245 if cfg.limited_deps:
246 the_data.setVar("BB_LIMITEDDEPS", "1")
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600247 the_data.setVar("BUILDNAME", workerdata["buildname"])
248 the_data.setVar("DATE", workerdata["date"])
249 the_data.setVar("TIME", workerdata["time"])
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500250 for varname, value in extraconfigdata.items():
251 the_data.setVar(varname, value)
252
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600253 bb.parse.siggen.set_taskdata(workerdata["sigdata"])
Brad Bishop08902b02019-08-20 09:16:51 -0400254 if "newhashes" in workerdata:
255 bb.parse.siggen.set_taskhashes(workerdata["newhashes"])
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600256 ret = 0
257
258 the_data = bb_cache.loadDataFull(fn, appends)
Brad Bishop19323692019-04-05 15:28:33 -0400259 the_data.setVar('BB_TASKHASH', taskhash)
260 the_data.setVar('BB_UNIHASH', unihash)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500261
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500262 bb.utils.set_process_name("%s:%s" % (the_data.getVar("PN"), taskname.replace("do_", "")))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500263
Andrew Geissler595f6302022-01-24 19:11:47 +0000264 if not the_data.getVarFlag(taskname, 'network', False):
265 logger.debug("Attempting to disable network")
266 bb.utils.disable_network(uid, gid)
267
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500268 # exported_vars() returns a generator which *cannot* be passed to os.environ.update()
269 # successfully. We also need to unset anything from the environment which shouldn't be there
270 exports = bb.data.exported_vars(the_data)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600271
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500272 bb.utils.empty_environment()
273 for e, v in exports:
274 os.environ[e] = v
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600275
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500276 for e in fakeenv:
277 os.environ[e] = fakeenv[e]
278 the_data.setVar(e, fakeenv[e])
279 the_data.setVarFlag(e, 'export', "1")
280
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500281 task_exports = the_data.getVarFlag(taskname, 'exports')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600282 if task_exports:
283 for e in task_exports.split():
284 the_data.setVarFlag(e, 'export', '1')
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500285 v = the_data.getVar(e)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600286 if v is not None:
287 os.environ[e] = v
288
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500289 if quieterrors:
290 the_data.setVarFlag(taskname, "quieterrors", "1")
291
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500292 except Exception:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500293 if not quieterrors:
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500294 logger.critical(traceback.format_exc())
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500295 os._exit(1)
296 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500297 if dry_run:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500298 return 0
Andrew Geisslereff27472021-10-29 15:35:00 -0500299 try:
300 ret = bb.build.exec_task(fn, taskname, the_data, cfg.profile)
301 finally:
302 if fakeroot:
303 fakerootcmd = shlex.split(the_data.getVar("FAKEROOTCMD"))
304 subprocess.run(fakerootcmd + ['-S'], check=True, stdout=subprocess.PIPE)
Patrick Williams93c203f2021-10-06 16:15:23 -0500305 return ret
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500306 except:
307 os._exit(1)
308 if not profiling:
309 os._exit(child())
310 else:
311 profname = "profile-%s.log" % (fn.replace("/", "-") + "-" + taskname)
312 prof = profile.Profile()
313 try:
314 ret = profile.Profile.runcall(prof, child)
315 finally:
316 prof.dump_stats(profname)
317 bb.utils.process_profilelog(profname)
318 os._exit(ret)
319 else:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600320 for key, value in iter(envbackup.items()):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500321 if value is None:
322 del os.environ[key]
323 else:
324 os.environ[key] = value
325
326 return pid, pipein, pipeout
327
328class runQueueWorkerPipe():
329 """
330 Abstraction for a pipe between a worker thread and the worker server
331 """
332 def __init__(self, pipein, pipeout):
333 self.input = pipein
334 if pipeout:
335 pipeout.close()
336 bb.utils.nonblockingfd(self.input)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600337 self.queue = b""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500338
339 def read(self):
340 start = len(self.queue)
341 try:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600342 self.queue = self.queue + (self.input.read(102400) or b"")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500343 except (OSError, IOError) as e:
344 if e.errno != errno.EAGAIN:
345 raise
346
347 end = len(self.queue)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600348 index = self.queue.find(b"</event>")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500349 while index != -1:
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600350 msg = self.queue[:index+8]
351 assert msg.startswith(b"<event>") and msg.count(b"<event>") == 1
352 worker_fire_prepickled(msg)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500353 self.queue = self.queue[index+8:]
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600354 index = self.queue.find(b"</event>")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500355 return (end > start)
356
357 def close(self):
358 while self.read():
359 continue
360 if len(self.queue) > 0:
361 print("Warning, worker child left partial message: %s" % self.queue)
362 self.input.close()
363
364normalexit = False
365
366class BitbakeWorker(object):
367 def __init__(self, din):
368 self.input = din
369 bb.utils.nonblockingfd(self.input)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600370 self.queue = b""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500371 self.cookercfg = None
372 self.databuilder = None
373 self.data = None
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500374 self.extraconfigdata = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500375 self.build_pids = {}
376 self.build_pipes = {}
377
378 signal.signal(signal.SIGTERM, self.sigterm_exception)
379 # Let SIGHUP exit as SIGTERM
380 signal.signal(signal.SIGHUP, self.sigterm_exception)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500381 if "beef" in sys.argv[1]:
382 bb.utils.set_process_name("Worker (Fakeroot)")
383 else:
384 bb.utils.set_process_name("Worker")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500385
386 def sigterm_exception(self, signum, stackframe):
387 if signum == signal.SIGTERM:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500388 bb.warn("Worker received SIGTERM, shutting down...")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500389 elif signum == signal.SIGHUP:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500390 bb.warn("Worker received SIGHUP, shutting down...")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500391 self.handle_finishnow(None)
392 signal.signal(signal.SIGTERM, signal.SIG_DFL)
393 os.kill(os.getpid(), signal.SIGTERM)
394
395 def serve(self):
396 while True:
397 (ready, _, _) = select.select([self.input] + [i.input for i in self.build_pipes.values()], [] , [], 1)
398 if self.input in ready:
399 try:
400 r = self.input.read()
401 if len(r) == 0:
402 # EOF on pipe, server must have terminated
403 self.sigterm_exception(signal.SIGTERM, None)
404 self.queue = self.queue + r
405 except (OSError, IOError):
406 pass
407 if len(self.queue):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600408 self.handle_item(b"cookerconfig", self.handle_cookercfg)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500409 self.handle_item(b"extraconfigdata", self.handle_extraconfigdata)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600410 self.handle_item(b"workerdata", self.handle_workerdata)
Brad Bishop08902b02019-08-20 09:16:51 -0400411 self.handle_item(b"newtaskhashes", self.handle_newtaskhashes)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600412 self.handle_item(b"runtask", self.handle_runtask)
413 self.handle_item(b"finishnow", self.handle_finishnow)
414 self.handle_item(b"ping", self.handle_ping)
415 self.handle_item(b"quit", self.handle_quit)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500416
417 for pipe in self.build_pipes:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500418 if self.build_pipes[pipe].input in ready:
419 self.build_pipes[pipe].read()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500420 if len(self.build_pids):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500421 while self.process_waitpid():
422 continue
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500423
424
425 def handle_item(self, item, func):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600426 if self.queue.startswith(b"<" + item + b">"):
427 index = self.queue.find(b"</" + item + b">")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500428 while index != -1:
Andrew Geisslereff27472021-10-29 15:35:00 -0500429 try:
430 func(self.queue[(len(item) + 2):index])
431 except pickle.UnpicklingError:
432 workerlog_write("Unable to unpickle data: %s\n" % ":".join("{:02x}".format(c) for c in self.queue))
433 raise
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500434 self.queue = self.queue[(index + len(item) + 3):]
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600435 index = self.queue.find(b"</" + item + b">")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500436
437 def handle_cookercfg(self, data):
438 self.cookercfg = pickle.loads(data)
439 self.databuilder = bb.cookerdata.CookerDataBuilder(self.cookercfg, worker=True)
440 self.databuilder.parseBaseConfiguration()
441 self.data = self.databuilder.data
442
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500443 def handle_extraconfigdata(self, data):
444 self.extraconfigdata = pickle.loads(data)
445
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500446 def handle_workerdata(self, data):
447 self.workerdata = pickle.loads(data)
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500448 bb.build.verboseShellLogging = self.workerdata["build_verbose_shell"]
449 bb.build.verboseStdoutLogging = self.workerdata["build_verbose_stdout"]
Andrew Geissler82c905d2020-04-13 13:39:40 -0500450 bb.msg.loggerDefaultLogLevel = self.workerdata["logdefaultlevel"]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500451 bb.msg.loggerDefaultDomains = self.workerdata["logdefaultdomain"]
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600452 for mc in self.databuilder.mcdata:
453 self.databuilder.mcdata[mc].setVar("PRSERV_HOST", self.workerdata["prhost"])
Brad Bishopa34c0302019-09-23 22:34:48 -0400454 self.databuilder.mcdata[mc].setVar("BB_HASHSERVE", self.workerdata["hashservaddr"])
Brad Bishop08902b02019-08-20 09:16:51 -0400455
456 def handle_newtaskhashes(self, data):
457 self.workerdata["newhashes"] = pickle.loads(data)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500458
459 def handle_ping(self, _):
460 workerlog_write("Handling ping\n")
461
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600462 logger.warning("Pong from bitbake-worker!")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500463
464 def handle_quit(self, data):
465 workerlog_write("Handling quit\n")
466
467 global normalexit
468 normalexit = True
469 sys.exit(0)
470
471 def handle_runtask(self, data):
Brad Bishop19323692019-04-05 15:28:33 -0400472 fn, task, taskname, taskhash, unihash, quieterrors, appends, taskdepdata, dry_run_exec = pickle.loads(data)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500473 workerlog_write("Handling runtask %s %s %s\n" % (task, fn, taskname))
474
Brad Bishop19323692019-04-05 15:28:33 -0400475 pid, pipein, pipeout = fork_off_task(self.cookercfg, self.data, self.databuilder, self.workerdata, fn, task, taskname, taskhash, unihash, appends, taskdepdata, self.extraconfigdata, quieterrors, dry_run_exec)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500476
477 self.build_pids[pid] = task
478 self.build_pipes[pid] = runQueueWorkerPipe(pipein, pipeout)
479
480 def process_waitpid(self):
481 """
482 Return none is there are no processes awaiting result collection, otherwise
483 collect the process exit codes and close the information pipe.
484 """
485 try:
486 pid, status = os.waitpid(-1, os.WNOHANG)
487 if pid == 0 or os.WIFSTOPPED(status):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500488 return False
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500489 except OSError:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500490 return False
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500491
492 workerlog_write("Exit code of %s for pid %s\n" % (status, pid))
493
494 if os.WIFEXITED(status):
495 status = os.WEXITSTATUS(status)
496 elif os.WIFSIGNALED(status):
497 # Per shell conventions for $?, when a process exits due to
498 # a signal, we return an exit code of 128 + SIGNUM
499 status = 128 + os.WTERMSIG(status)
500
501 task = self.build_pids[pid]
502 del self.build_pids[pid]
503
504 self.build_pipes[pid].close()
505 del self.build_pipes[pid]
506
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600507 worker_fire_prepickled(b"<exitcode>" + pickle.dumps((task, status)) + b"</exitcode>")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500508
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500509 return True
510
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500511 def handle_finishnow(self, _):
512 if self.build_pids:
513 logger.info("Sending SIGTERM to remaining %s tasks", len(self.build_pids))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600514 for k, v in iter(self.build_pids.items()):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500515 try:
516 os.kill(-k, signal.SIGTERM)
517 os.waitpid(-1, 0)
518 except:
519 pass
520 for pipe in self.build_pipes:
521 self.build_pipes[pipe].read()
522
523try:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600524 worker = BitbakeWorker(os.fdopen(sys.stdin.fileno(), 'rb'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500525 if not profiling:
526 worker.serve()
527 else:
528 profname = "profile-worker.log"
529 prof = profile.Profile()
530 try:
531 profile.Profile.runcall(prof, worker.serve)
532 finally:
533 prof.dump_stats(profname)
534 bb.utils.process_profilelog(profname)
535except BaseException as e:
536 if not normalexit:
537 import traceback
538 sys.stderr.write(traceback.format_exc())
539 sys.stderr.write(str(e))
Andrew Geissler5199d832021-09-24 16:47:35 -0500540finally:
541 worker_thread_exit = True
542 worker_thread.join()
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500543
Andrew Geisslerd159c7f2021-09-02 21:05:58 -0500544workerlog_write("exiting")
Andrew Geissler5199d832021-09-24 16:47:35 -0500545if not normalexit:
546 sys.exit(1)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500547sys.exit(0)