blob: a3ea5d961865f14e3f439c664baf83d8240015dd [file] [log] [blame]
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001#!/usr/bin/env python3
Brad Bishopc342db32019-05-15 21:57:59 -04002#
Patrick Williams92b42cb2022-09-03 06:53:57 -05003# Copyright BitBake Contributors
4#
Brad Bishopc342db32019-05-15 21:57:59 -04005# SPDX-License-Identifier: GPL-2.0-only
6#
Patrick Williamsc124f4f2015-09-15 14:41:29 -05007
8import os
9import sys
10import warnings
Andrew Geissler5199d832021-09-24 16:47:35 -050011warnings.simplefilter("default")
Patrick Williamsc124f4f2015-09-15 14:41:29 -050012sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
13from bb import fetch2
14import logging
15import bb
16import select
17import errno
18import signal
Patrick Williamsc0f7c042017-02-23 20:41:17 -060019import pickle
Brad Bishop37a0e4d2017-12-04 01:01:44 -050020import traceback
21import queue
Patrick Williams93c203f2021-10-06 16:15:23 -050022import shlex
23import subprocess
Patrick Williamsf1e5d692016-03-30 15:21:19 -050024from multiprocessing import Lock
Brad Bishop37a0e4d2017-12-04 01:01:44 -050025from threading import Thread
Patrick Williamsc124f4f2015-09-15 14:41:29 -050026
Andrew Geissler517393d2023-01-13 08:55:19 -060027bb.utils.check_system_locale()
Patrick Williamsc0f7c042017-02-23 20:41:17 -060028
Patrick Williamsc124f4f2015-09-15 14:41:29 -050029# Users shouldn't be running this code directly
30if len(sys.argv) != 2 or not sys.argv[1].startswith("decafbad"):
31 print("bitbake-worker is meant for internal execution by bitbake itself, please don't use it standalone.")
32 sys.exit(1)
33
34profiling = False
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050035if sys.argv[1].startswith("decafbadbad"):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050036 profiling = True
37 try:
38 import cProfile as profile
39 except:
40 import profile
41
42# Unbuffer stdout to avoid log truncation in the event
43# of an unorderly exit as well as to provide timely
44# updates to log files for use with tail
45try:
46 if sys.stdout.name == '<stdout>':
Patrick Williamsc0f7c042017-02-23 20:41:17 -060047 import fcntl
48 fl = fcntl.fcntl(sys.stdout.fileno(), fcntl.F_GETFL)
49 fl |= os.O_SYNC
50 fcntl.fcntl(sys.stdout.fileno(), fcntl.F_SETFL, fl)
51 #sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050052except:
53 pass
54
55logger = logging.getLogger("BitBake")
56
Patrick Williamsc124f4f2015-09-15 14:41:29 -050057worker_pipe = sys.stdout.fileno()
58bb.utils.nonblockingfd(worker_pipe)
Patrick Williamsf1e5d692016-03-30 15:21:19 -050059# Need to guard against multiprocessing being used in child processes
60# and multiple processes trying to write to the parent at the same time
61worker_pipe_lock = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -050062
63handler = bb.event.LogHandler()
64logger.addHandler(handler)
65
66if 0:
67 # Code to write out a log file of all events passing through the worker
68 logfilename = "/tmp/workerlogfile"
69 format_str = "%(levelname)s: %(message)s"
70 conlogformat = bb.msg.BBLogFormatter(format_str)
71 consolelog = logging.FileHandler(logfilename)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050072 consolelog.setFormatter(conlogformat)
73 logger.addHandler(consolelog)
74
Brad Bishop37a0e4d2017-12-04 01:01:44 -050075worker_queue = queue.Queue()
Patrick Williamsc124f4f2015-09-15 14:41:29 -050076
77def worker_fire(event, d):
Patrick Williamsc0f7c042017-02-23 20:41:17 -060078 data = b"<event>" + pickle.dumps(event) + b"</event>"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050079 worker_fire_prepickled(data)
80
81def worker_fire_prepickled(event):
82 global worker_queue
83
Brad Bishop37a0e4d2017-12-04 01:01:44 -050084 worker_queue.put(event)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050085
Brad Bishop37a0e4d2017-12-04 01:01:44 -050086#
87# We can end up with write contention with the cooker, it can be trying to send commands
88# and we can be trying to send event data back. Therefore use a separate thread for writing
89# back data to cooker.
90#
91worker_thread_exit = False
Patrick Williamsc124f4f2015-09-15 14:41:29 -050092
Brad Bishop37a0e4d2017-12-04 01:01:44 -050093def worker_flush(worker_queue):
94 worker_queue_int = b""
95 global worker_pipe, worker_thread_exit
Patrick Williamsc124f4f2015-09-15 14:41:29 -050096
Brad Bishop37a0e4d2017-12-04 01:01:44 -050097 while True:
98 try:
99 worker_queue_int = worker_queue_int + worker_queue.get(True, 1)
100 except queue.Empty:
101 pass
102 while (worker_queue_int or not worker_queue.empty()):
103 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500104 (_, ready, _) = select.select([], [worker_pipe], [], 1)
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500105 if not worker_queue.empty():
106 worker_queue_int = worker_queue_int + worker_queue.get()
107 written = os.write(worker_pipe, worker_queue_int)
108 worker_queue_int = worker_queue_int[written:]
109 except (IOError, OSError) as e:
110 if e.errno != errno.EAGAIN and e.errno != errno.EPIPE:
111 raise
112 if worker_thread_exit and worker_queue.empty() and not worker_queue_int:
113 return
114
115worker_thread = Thread(target=worker_flush, args=(worker_queue,))
116worker_thread.start()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500117
118def worker_child_fire(event, d):
119 global worker_pipe
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500120 global worker_pipe_lock
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500121
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600122 data = b"<event>" + pickle.dumps(event) + b"</event>"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500123 try:
Andrew Geissler517393d2023-01-13 08:55:19 -0600124 with bb.utils.lock_timeout(worker_pipe_lock):
125 while(len(data)):
126 written = worker_pipe.write(data)
127 data = data[written:]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500128 except IOError:
129 sigterm_handler(None, None)
130 raise
131
132bb.event.worker_fire = worker_fire
133
134lf = None
135#lf = open("/tmp/workercommandlog", "w+")
136def workerlog_write(msg):
137 if lf:
138 lf.write(msg)
139 lf.flush()
140
141def sigterm_handler(signum, frame):
142 signal.signal(signal.SIGTERM, signal.SIG_DFL)
143 os.killpg(0, signal.SIGTERM)
144 sys.exit()
145
Andrew Geissler517393d2023-01-13 08:55:19 -0600146def fork_off_task(cfg, data, databuilder, workerdata, extraconfigdata, runtask):
147
148 fn = runtask['fn']
149 task = runtask['task']
150 taskname = runtask['taskname']
151 taskhash = runtask['taskhash']
152 unihash = runtask['unihash']
153 appends = runtask['appends']
154 taskdepdata = runtask['taskdepdata']
155 quieterrors = runtask['quieterrors']
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500156 # We need to setup the environment BEFORE the fork, since
157 # a fork() or exec*() activates PSEUDO...
158
159 envbackup = {}
Patrick Williams93c203f2021-10-06 16:15:23 -0500160 fakeroot = False
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500161 fakeenv = {}
162 umask = None
163
Andrew Geissler595f6302022-01-24 19:11:47 +0000164 uid = os.getuid()
165 gid = os.getgid()
166
Andrew Geissler517393d2023-01-13 08:55:19 -0600167 taskdep = runtask['taskdep']
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500168 if 'umask' in taskdep and taskname in taskdep['umask']:
Andrew Geissler9b4d8b02021-02-19 12:26:16 -0600169 umask = taskdep['umask'][taskname]
170 elif workerdata["umask"]:
171 umask = workerdata["umask"]
172 if umask:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500173 # umask might come in as a number or text string..
174 try:
Andrew Geissler9b4d8b02021-02-19 12:26:16 -0600175 umask = int(umask, 8)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500176 except TypeError:
Andrew Geissler9b4d8b02021-02-19 12:26:16 -0600177 pass
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500178
Andrew Geissler517393d2023-01-13 08:55:19 -0600179 dry_run = cfg.dry_run or runtask['dry_run']
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500180
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500181 # We can't use the fakeroot environment in a dry run as it possibly hasn't been built
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500182 if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not dry_run:
Patrick Williams93c203f2021-10-06 16:15:23 -0500183 fakeroot = True
Andrew Geissler517393d2023-01-13 08:55:19 -0600184 envvars = (runtask['fakerootenv'] or "").split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500185 for key, value in (var.split('=') for var in envvars):
186 envbackup[key] = os.environ.get(key)
187 os.environ[key] = value
188 fakeenv[key] = value
189
Andrew Geissler517393d2023-01-13 08:55:19 -0600190 fakedirs = (runtask['fakerootdirs'] or "").split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500191 for p in fakedirs:
192 bb.utils.mkdirhier(p)
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600193 logger.debug2('Running %s:%s under fakeroot, fakedirs: %s' %
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500194 (fn, taskname, ', '.join(fakedirs)))
195 else:
Andrew Geissler517393d2023-01-13 08:55:19 -0600196 envvars = (runtask['fakerootnoenv'] or "").split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500197 for key, value in (var.split('=') for var in envvars):
198 envbackup[key] = os.environ.get(key)
199 os.environ[key] = value
200 fakeenv[key] = value
201
202 sys.stdout.flush()
203 sys.stderr.flush()
204
205 try:
206 pipein, pipeout = os.pipe()
207 pipein = os.fdopen(pipein, 'rb', 4096)
208 pipeout = os.fdopen(pipeout, 'wb', 0)
209 pid = os.fork()
210 except OSError as e:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600211 logger.critical("fork failed: %d (%s)" % (e.errno, e.strerror))
212 sys.exit(1)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500213
214 if pid == 0:
215 def child():
216 global worker_pipe
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500217 global worker_pipe_lock
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500218 pipein.close()
219
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500220 bb.utils.signal_on_parent_exit("SIGTERM")
221
222 # Save out the PID so that the event can include it the
223 # events
224 bb.event.worker_pid = os.getpid()
225 bb.event.worker_fire = worker_child_fire
226 worker_pipe = pipeout
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500227 worker_pipe_lock = Lock()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500228
229 # Make the child the process group leader and ensure no
230 # child process will be controlled by the current terminal
231 # This ensures signals sent to the controlling terminal like Ctrl+C
232 # don't stop the child processes.
233 os.setsid()
Brad Bishop1d80a2e2019-11-15 16:35:03 -0500234
235 signal.signal(signal.SIGTERM, sigterm_handler)
236 # Let SIGHUP exit as SIGTERM
237 signal.signal(signal.SIGHUP, sigterm_handler)
238
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500239 # No stdin
240 newsi = os.open(os.devnull, os.O_RDWR)
241 os.dup2(newsi, sys.stdin.fileno())
242
243 if umask:
244 os.umask(umask)
245
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500246 try:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600247 (realfn, virtual, mc) = bb.cache.virtualfn2realfn(fn)
248 the_data = databuilder.mcdata[mc]
249 the_data.setVar("BB_WORKERCONTEXT", "1")
250 the_data.setVar("BB_TASKDEPDATA", taskdepdata)
Andrew Geisslereff27472021-10-29 15:35:00 -0500251 the_data.setVar('BB_CURRENTTASK', taskname.replace("do_", ""))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500252 if cfg.limited_deps:
253 the_data.setVar("BB_LIMITEDDEPS", "1")
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600254 the_data.setVar("BUILDNAME", workerdata["buildname"])
255 the_data.setVar("DATE", workerdata["date"])
256 the_data.setVar("TIME", workerdata["time"])
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500257 for varname, value in extraconfigdata.items():
258 the_data.setVar(varname, value)
259
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600260 bb.parse.siggen.set_taskdata(workerdata["sigdata"])
Brad Bishop08902b02019-08-20 09:16:51 -0400261 if "newhashes" in workerdata:
262 bb.parse.siggen.set_taskhashes(workerdata["newhashes"])
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600263 ret = 0
264
Andrew Geissler517393d2023-01-13 08:55:19 -0600265 the_data = databuilder.parseRecipe(fn, appends)
Brad Bishop19323692019-04-05 15:28:33 -0400266 the_data.setVar('BB_TASKHASH', taskhash)
267 the_data.setVar('BB_UNIHASH', unihash)
Andrew Geissler517393d2023-01-13 08:55:19 -0600268 bb.parse.siggen.setup_datacache_from_datastore(fn, the_data)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500269
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500270 bb.utils.set_process_name("%s:%s" % (the_data.getVar("PN"), taskname.replace("do_", "")))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500271
Andrew Geissler595f6302022-01-24 19:11:47 +0000272 if not the_data.getVarFlag(taskname, 'network', False):
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000273 if bb.utils.is_local_uid(uid):
274 logger.debug("Attempting to disable network for %s" % taskname)
275 bb.utils.disable_network(uid, gid)
276 else:
277 logger.debug("Skipping disable network for %s since %s is not a local uid." % (taskname, uid))
Andrew Geissler595f6302022-01-24 19:11:47 +0000278
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500279 # exported_vars() returns a generator which *cannot* be passed to os.environ.update()
280 # successfully. We also need to unset anything from the environment which shouldn't be there
281 exports = bb.data.exported_vars(the_data)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600282
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500283 bb.utils.empty_environment()
284 for e, v in exports:
285 os.environ[e] = v
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600286
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500287 for e in fakeenv:
288 os.environ[e] = fakeenv[e]
289 the_data.setVar(e, fakeenv[e])
290 the_data.setVarFlag(e, 'export', "1")
291
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500292 task_exports = the_data.getVarFlag(taskname, 'exports')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600293 if task_exports:
294 for e in task_exports.split():
295 the_data.setVarFlag(e, 'export', '1')
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500296 v = the_data.getVar(e)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600297 if v is not None:
298 os.environ[e] = v
299
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500300 if quieterrors:
301 the_data.setVarFlag(taskname, "quieterrors", "1")
302
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500303 except Exception:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500304 if not quieterrors:
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500305 logger.critical(traceback.format_exc())
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500306 os._exit(1)
307 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500308 if dry_run:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500309 return 0
Andrew Geisslereff27472021-10-29 15:35:00 -0500310 try:
311 ret = bb.build.exec_task(fn, taskname, the_data, cfg.profile)
312 finally:
313 if fakeroot:
314 fakerootcmd = shlex.split(the_data.getVar("FAKEROOTCMD"))
315 subprocess.run(fakerootcmd + ['-S'], check=True, stdout=subprocess.PIPE)
Patrick Williams93c203f2021-10-06 16:15:23 -0500316 return ret
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500317 except:
318 os._exit(1)
319 if not profiling:
320 os._exit(child())
321 else:
322 profname = "profile-%s.log" % (fn.replace("/", "-") + "-" + taskname)
323 prof = profile.Profile()
324 try:
325 ret = profile.Profile.runcall(prof, child)
326 finally:
327 prof.dump_stats(profname)
328 bb.utils.process_profilelog(profname)
329 os._exit(ret)
330 else:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600331 for key, value in iter(envbackup.items()):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500332 if value is None:
333 del os.environ[key]
334 else:
335 os.environ[key] = value
336
337 return pid, pipein, pipeout
338
339class runQueueWorkerPipe():
340 """
341 Abstraction for a pipe between a worker thread and the worker server
342 """
343 def __init__(self, pipein, pipeout):
344 self.input = pipein
345 if pipeout:
346 pipeout.close()
347 bb.utils.nonblockingfd(self.input)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600348 self.queue = b""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500349
350 def read(self):
351 start = len(self.queue)
352 try:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600353 self.queue = self.queue + (self.input.read(102400) or b"")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500354 except (OSError, IOError) as e:
355 if e.errno != errno.EAGAIN:
356 raise
357
358 end = len(self.queue)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600359 index = self.queue.find(b"</event>")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500360 while index != -1:
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600361 msg = self.queue[:index+8]
362 assert msg.startswith(b"<event>") and msg.count(b"<event>") == 1
363 worker_fire_prepickled(msg)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500364 self.queue = self.queue[index+8:]
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600365 index = self.queue.find(b"</event>")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500366 return (end > start)
367
368 def close(self):
369 while self.read():
370 continue
371 if len(self.queue) > 0:
372 print("Warning, worker child left partial message: %s" % self.queue)
373 self.input.close()
374
375normalexit = False
376
377class BitbakeWorker(object):
378 def __init__(self, din):
379 self.input = din
380 bb.utils.nonblockingfd(self.input)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600381 self.queue = b""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500382 self.cookercfg = None
383 self.databuilder = None
384 self.data = None
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500385 self.extraconfigdata = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500386 self.build_pids = {}
387 self.build_pipes = {}
388
389 signal.signal(signal.SIGTERM, self.sigterm_exception)
390 # Let SIGHUP exit as SIGTERM
391 signal.signal(signal.SIGHUP, self.sigterm_exception)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500392 if "beef" in sys.argv[1]:
393 bb.utils.set_process_name("Worker (Fakeroot)")
394 else:
395 bb.utils.set_process_name("Worker")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500396
397 def sigterm_exception(self, signum, stackframe):
398 if signum == signal.SIGTERM:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500399 bb.warn("Worker received SIGTERM, shutting down...")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500400 elif signum == signal.SIGHUP:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500401 bb.warn("Worker received SIGHUP, shutting down...")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500402 self.handle_finishnow(None)
403 signal.signal(signal.SIGTERM, signal.SIG_DFL)
404 os.kill(os.getpid(), signal.SIGTERM)
405
406 def serve(self):
407 while True:
408 (ready, _, _) = select.select([self.input] + [i.input for i in self.build_pipes.values()], [] , [], 1)
409 if self.input in ready:
410 try:
411 r = self.input.read()
412 if len(r) == 0:
413 # EOF on pipe, server must have terminated
414 self.sigterm_exception(signal.SIGTERM, None)
415 self.queue = self.queue + r
416 except (OSError, IOError):
417 pass
418 if len(self.queue):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600419 self.handle_item(b"cookerconfig", self.handle_cookercfg)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500420 self.handle_item(b"extraconfigdata", self.handle_extraconfigdata)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600421 self.handle_item(b"workerdata", self.handle_workerdata)
Brad Bishop08902b02019-08-20 09:16:51 -0400422 self.handle_item(b"newtaskhashes", self.handle_newtaskhashes)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600423 self.handle_item(b"runtask", self.handle_runtask)
424 self.handle_item(b"finishnow", self.handle_finishnow)
425 self.handle_item(b"ping", self.handle_ping)
426 self.handle_item(b"quit", self.handle_quit)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500427
428 for pipe in self.build_pipes:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500429 if self.build_pipes[pipe].input in ready:
430 self.build_pipes[pipe].read()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500431 if len(self.build_pids):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500432 while self.process_waitpid():
433 continue
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500434
435
436 def handle_item(self, item, func):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600437 if self.queue.startswith(b"<" + item + b">"):
438 index = self.queue.find(b"</" + item + b">")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500439 while index != -1:
Andrew Geisslereff27472021-10-29 15:35:00 -0500440 try:
441 func(self.queue[(len(item) + 2):index])
442 except pickle.UnpicklingError:
443 workerlog_write("Unable to unpickle data: %s\n" % ":".join("{:02x}".format(c) for c in self.queue))
444 raise
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500445 self.queue = self.queue[(index + len(item) + 3):]
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600446 index = self.queue.find(b"</" + item + b">")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500447
448 def handle_cookercfg(self, data):
449 self.cookercfg = pickle.loads(data)
450 self.databuilder = bb.cookerdata.CookerDataBuilder(self.cookercfg, worker=True)
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000451 self.databuilder.parseBaseConfiguration(worker=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500452 self.data = self.databuilder.data
453
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500454 def handle_extraconfigdata(self, data):
455 self.extraconfigdata = pickle.loads(data)
456
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500457 def handle_workerdata(self, data):
458 self.workerdata = pickle.loads(data)
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500459 bb.build.verboseShellLogging = self.workerdata["build_verbose_shell"]
460 bb.build.verboseStdoutLogging = self.workerdata["build_verbose_stdout"]
Andrew Geissler82c905d2020-04-13 13:39:40 -0500461 bb.msg.loggerDefaultLogLevel = self.workerdata["logdefaultlevel"]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500462 bb.msg.loggerDefaultDomains = self.workerdata["logdefaultdomain"]
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600463 for mc in self.databuilder.mcdata:
464 self.databuilder.mcdata[mc].setVar("PRSERV_HOST", self.workerdata["prhost"])
Brad Bishopa34c0302019-09-23 22:34:48 -0400465 self.databuilder.mcdata[mc].setVar("BB_HASHSERVE", self.workerdata["hashservaddr"])
Patrick Williams92b42cb2022-09-03 06:53:57 -0500466 self.databuilder.mcdata[mc].setVar("__bbclasstype", "recipe")
Brad Bishop08902b02019-08-20 09:16:51 -0400467
468 def handle_newtaskhashes(self, data):
469 self.workerdata["newhashes"] = pickle.loads(data)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500470
471 def handle_ping(self, _):
472 workerlog_write("Handling ping\n")
473
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600474 logger.warning("Pong from bitbake-worker!")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500475
476 def handle_quit(self, data):
477 workerlog_write("Handling quit\n")
478
479 global normalexit
480 normalexit = True
481 sys.exit(0)
482
483 def handle_runtask(self, data):
Andrew Geissler517393d2023-01-13 08:55:19 -0600484 runtask = pickle.loads(data)
485
486 fn = runtask['fn']
487 task = runtask['task']
488 taskname = runtask['taskname']
489
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500490 workerlog_write("Handling runtask %s %s %s\n" % (task, fn, taskname))
491
Andrew Geissler517393d2023-01-13 08:55:19 -0600492 pid, pipein, pipeout = fork_off_task(self.cookercfg, self.data, self.databuilder, self.workerdata, self.extraconfigdata, runtask)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500493 self.build_pids[pid] = task
494 self.build_pipes[pid] = runQueueWorkerPipe(pipein, pipeout)
495
496 def process_waitpid(self):
497 """
498 Return none is there are no processes awaiting result collection, otherwise
499 collect the process exit codes and close the information pipe.
500 """
501 try:
502 pid, status = os.waitpid(-1, os.WNOHANG)
503 if pid == 0 or os.WIFSTOPPED(status):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500504 return False
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500505 except OSError:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500506 return False
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500507
508 workerlog_write("Exit code of %s for pid %s\n" % (status, pid))
509
510 if os.WIFEXITED(status):
511 status = os.WEXITSTATUS(status)
512 elif os.WIFSIGNALED(status):
513 # Per shell conventions for $?, when a process exits due to
514 # a signal, we return an exit code of 128 + SIGNUM
515 status = 128 + os.WTERMSIG(status)
516
517 task = self.build_pids[pid]
518 del self.build_pids[pid]
519
520 self.build_pipes[pid].close()
521 del self.build_pipes[pid]
522
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600523 worker_fire_prepickled(b"<exitcode>" + pickle.dumps((task, status)) + b"</exitcode>")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500524
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500525 return True
526
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500527 def handle_finishnow(self, _):
528 if self.build_pids:
529 logger.info("Sending SIGTERM to remaining %s tasks", len(self.build_pids))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600530 for k, v in iter(self.build_pids.items()):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500531 try:
532 os.kill(-k, signal.SIGTERM)
533 os.waitpid(-1, 0)
534 except:
535 pass
536 for pipe in self.build_pipes:
537 self.build_pipes[pipe].read()
538
539try:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600540 worker = BitbakeWorker(os.fdopen(sys.stdin.fileno(), 'rb'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500541 if not profiling:
542 worker.serve()
543 else:
544 profname = "profile-worker.log"
545 prof = profile.Profile()
546 try:
547 profile.Profile.runcall(prof, worker.serve)
548 finally:
549 prof.dump_stats(profname)
550 bb.utils.process_profilelog(profname)
551except BaseException as e:
552 if not normalexit:
553 import traceback
554 sys.stderr.write(traceback.format_exc())
555 sys.stderr.write(str(e))
Andrew Geissler5199d832021-09-24 16:47:35 -0500556finally:
557 worker_thread_exit = True
558 worker_thread.join()
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500559
Andrew Geisslerd159c7f2021-09-02 21:05:58 -0500560workerlog_write("exiting")
Andrew Geissler5199d832021-09-24 16:47:35 -0500561if not normalexit:
562 sys.exit(1)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500563sys.exit(0)