blob: 609e276fe2d4789a2a04a83e8e0ceb564325e41b [file] [log] [blame]
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001#!/usr/bin/env python3
Brad Bishopc342db32019-05-15 21:57:59 -04002#
Patrick Williams92b42cb2022-09-03 06:53:57 -05003# Copyright BitBake Contributors
4#
Brad Bishopc342db32019-05-15 21:57:59 -04005# SPDX-License-Identifier: GPL-2.0-only
6#
Patrick Williamsc124f4f2015-09-15 14:41:29 -05007
8import os
9import sys
10import warnings
Andrew Geissler5199d832021-09-24 16:47:35 -050011warnings.simplefilter("default")
Patrick Williamsc124f4f2015-09-15 14:41:29 -050012sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
13from bb import fetch2
14import logging
15import bb
16import select
17import errno
18import signal
Patrick Williamsc0f7c042017-02-23 20:41:17 -060019import pickle
Brad Bishop37a0e4d2017-12-04 01:01:44 -050020import traceback
21import queue
Patrick Williams93c203f2021-10-06 16:15:23 -050022import shlex
23import subprocess
Patrick Williamsf1e5d692016-03-30 15:21:19 -050024from multiprocessing import Lock
Brad Bishop37a0e4d2017-12-04 01:01:44 -050025from threading import Thread
Patrick Williamsc124f4f2015-09-15 14:41:29 -050026
Andrew Geissler517393d2023-01-13 08:55:19 -060027bb.utils.check_system_locale()
Patrick Williamsc0f7c042017-02-23 20:41:17 -060028
Patrick Williamsc124f4f2015-09-15 14:41:29 -050029# Users shouldn't be running this code directly
30if len(sys.argv) != 2 or not sys.argv[1].startswith("decafbad"):
31 print("bitbake-worker is meant for internal execution by bitbake itself, please don't use it standalone.")
32 sys.exit(1)
33
34profiling = False
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050035if sys.argv[1].startswith("decafbadbad"):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050036 profiling = True
37 try:
38 import cProfile as profile
39 except:
40 import profile
41
42# Unbuffer stdout to avoid log truncation in the event
43# of an unorderly exit as well as to provide timely
44# updates to log files for use with tail
45try:
46 if sys.stdout.name == '<stdout>':
Patrick Williamsc0f7c042017-02-23 20:41:17 -060047 import fcntl
48 fl = fcntl.fcntl(sys.stdout.fileno(), fcntl.F_GETFL)
49 fl |= os.O_SYNC
50 fcntl.fcntl(sys.stdout.fileno(), fcntl.F_SETFL, fl)
51 #sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050052except:
53 pass
54
55logger = logging.getLogger("BitBake")
56
Patrick Williamsc124f4f2015-09-15 14:41:29 -050057worker_pipe = sys.stdout.fileno()
58bb.utils.nonblockingfd(worker_pipe)
Patrick Williamsf1e5d692016-03-30 15:21:19 -050059# Need to guard against multiprocessing being used in child processes
60# and multiple processes trying to write to the parent at the same time
61worker_pipe_lock = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -050062
63handler = bb.event.LogHandler()
64logger.addHandler(handler)
65
66if 0:
67 # Code to write out a log file of all events passing through the worker
68 logfilename = "/tmp/workerlogfile"
69 format_str = "%(levelname)s: %(message)s"
70 conlogformat = bb.msg.BBLogFormatter(format_str)
71 consolelog = logging.FileHandler(logfilename)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050072 consolelog.setFormatter(conlogformat)
73 logger.addHandler(consolelog)
74
Brad Bishop37a0e4d2017-12-04 01:01:44 -050075worker_queue = queue.Queue()
Patrick Williamsc124f4f2015-09-15 14:41:29 -050076
77def worker_fire(event, d):
Patrick Williamsc0f7c042017-02-23 20:41:17 -060078 data = b"<event>" + pickle.dumps(event) + b"</event>"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050079 worker_fire_prepickled(data)
80
81def worker_fire_prepickled(event):
82 global worker_queue
83
Brad Bishop37a0e4d2017-12-04 01:01:44 -050084 worker_queue.put(event)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050085
Brad Bishop37a0e4d2017-12-04 01:01:44 -050086#
87# We can end up with write contention with the cooker, it can be trying to send commands
88# and we can be trying to send event data back. Therefore use a separate thread for writing
89# back data to cooker.
90#
91worker_thread_exit = False
Patrick Williamsc124f4f2015-09-15 14:41:29 -050092
Brad Bishop37a0e4d2017-12-04 01:01:44 -050093def worker_flush(worker_queue):
Andrew Geissler220dafd2023-10-04 10:18:08 -050094 worker_queue_int = bytearray()
Brad Bishop37a0e4d2017-12-04 01:01:44 -050095 global worker_pipe, worker_thread_exit
Patrick Williamsc124f4f2015-09-15 14:41:29 -050096
Brad Bishop37a0e4d2017-12-04 01:01:44 -050097 while True:
98 try:
Andrew Geissler220dafd2023-10-04 10:18:08 -050099 worker_queue_int.extend(worker_queue.get(True, 1))
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500100 except queue.Empty:
101 pass
102 while (worker_queue_int or not worker_queue.empty()):
103 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500104 (_, ready, _) = select.select([], [worker_pipe], [], 1)
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500105 if not worker_queue.empty():
Andrew Geissler220dafd2023-10-04 10:18:08 -0500106 worker_queue_int.extend(worker_queue.get())
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500107 written = os.write(worker_pipe, worker_queue_int)
108 worker_queue_int = worker_queue_int[written:]
109 except (IOError, OSError) as e:
110 if e.errno != errno.EAGAIN and e.errno != errno.EPIPE:
111 raise
112 if worker_thread_exit and worker_queue.empty() and not worker_queue_int:
113 return
114
115worker_thread = Thread(target=worker_flush, args=(worker_queue,))
116worker_thread.start()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500117
118def worker_child_fire(event, d):
119 global worker_pipe
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500120 global worker_pipe_lock
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500121
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600122 data = b"<event>" + pickle.dumps(event) + b"</event>"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500123 try:
Andrew Geissler517393d2023-01-13 08:55:19 -0600124 with bb.utils.lock_timeout(worker_pipe_lock):
125 while(len(data)):
126 written = worker_pipe.write(data)
127 data = data[written:]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500128 except IOError:
129 sigterm_handler(None, None)
130 raise
131
132bb.event.worker_fire = worker_fire
133
134lf = None
135#lf = open("/tmp/workercommandlog", "w+")
136def workerlog_write(msg):
137 if lf:
138 lf.write(msg)
139 lf.flush()
140
141def sigterm_handler(signum, frame):
142 signal.signal(signal.SIGTERM, signal.SIG_DFL)
143 os.killpg(0, signal.SIGTERM)
144 sys.exit()
145
Andrew Geissler517393d2023-01-13 08:55:19 -0600146def fork_off_task(cfg, data, databuilder, workerdata, extraconfigdata, runtask):
147
148 fn = runtask['fn']
149 task = runtask['task']
150 taskname = runtask['taskname']
151 taskhash = runtask['taskhash']
152 unihash = runtask['unihash']
153 appends = runtask['appends']
Patrick Williamse760df82023-05-26 11:10:49 -0500154 layername = runtask['layername']
Andrew Geissler517393d2023-01-13 08:55:19 -0600155 taskdepdata = runtask['taskdepdata']
156 quieterrors = runtask['quieterrors']
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500157 # We need to setup the environment BEFORE the fork, since
158 # a fork() or exec*() activates PSEUDO...
159
160 envbackup = {}
Patrick Williams93c203f2021-10-06 16:15:23 -0500161 fakeroot = False
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500162 fakeenv = {}
163 umask = None
164
Andrew Geissler595f6302022-01-24 19:11:47 +0000165 uid = os.getuid()
166 gid = os.getgid()
167
Andrew Geissler517393d2023-01-13 08:55:19 -0600168 taskdep = runtask['taskdep']
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500169 if 'umask' in taskdep and taskname in taskdep['umask']:
Andrew Geissler9b4d8b02021-02-19 12:26:16 -0600170 umask = taskdep['umask'][taskname]
171 elif workerdata["umask"]:
172 umask = workerdata["umask"]
173 if umask:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500174 # umask might come in as a number or text string..
175 try:
Andrew Geissler9b4d8b02021-02-19 12:26:16 -0600176 umask = int(umask, 8)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500177 except TypeError:
Andrew Geissler9b4d8b02021-02-19 12:26:16 -0600178 pass
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500179
Andrew Geissler517393d2023-01-13 08:55:19 -0600180 dry_run = cfg.dry_run or runtask['dry_run']
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500181
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500182 # We can't use the fakeroot environment in a dry run as it possibly hasn't been built
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500183 if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not dry_run:
Patrick Williams93c203f2021-10-06 16:15:23 -0500184 fakeroot = True
Andrew Geissler517393d2023-01-13 08:55:19 -0600185 envvars = (runtask['fakerootenv'] or "").split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500186 for key, value in (var.split('=') for var in envvars):
187 envbackup[key] = os.environ.get(key)
188 os.environ[key] = value
189 fakeenv[key] = value
190
Andrew Geissler517393d2023-01-13 08:55:19 -0600191 fakedirs = (runtask['fakerootdirs'] or "").split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500192 for p in fakedirs:
193 bb.utils.mkdirhier(p)
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600194 logger.debug2('Running %s:%s under fakeroot, fakedirs: %s' %
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500195 (fn, taskname, ', '.join(fakedirs)))
196 else:
Andrew Geissler517393d2023-01-13 08:55:19 -0600197 envvars = (runtask['fakerootnoenv'] or "").split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500198 for key, value in (var.split('=') for var in envvars):
199 envbackup[key] = os.environ.get(key)
200 os.environ[key] = value
201 fakeenv[key] = value
202
203 sys.stdout.flush()
204 sys.stderr.flush()
205
206 try:
207 pipein, pipeout = os.pipe()
208 pipein = os.fdopen(pipein, 'rb', 4096)
209 pipeout = os.fdopen(pipeout, 'wb', 0)
210 pid = os.fork()
211 except OSError as e:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600212 logger.critical("fork failed: %d (%s)" % (e.errno, e.strerror))
213 sys.exit(1)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500214
215 if pid == 0:
216 def child():
217 global worker_pipe
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500218 global worker_pipe_lock
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500219 pipein.close()
220
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500221 bb.utils.signal_on_parent_exit("SIGTERM")
222
223 # Save out the PID so that the event can include it the
224 # events
225 bb.event.worker_pid = os.getpid()
226 bb.event.worker_fire = worker_child_fire
227 worker_pipe = pipeout
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500228 worker_pipe_lock = Lock()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500229
230 # Make the child the process group leader and ensure no
231 # child process will be controlled by the current terminal
232 # This ensures signals sent to the controlling terminal like Ctrl+C
233 # don't stop the child processes.
234 os.setsid()
Brad Bishop1d80a2e2019-11-15 16:35:03 -0500235
236 signal.signal(signal.SIGTERM, sigterm_handler)
237 # Let SIGHUP exit as SIGTERM
238 signal.signal(signal.SIGHUP, sigterm_handler)
239
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500240 # No stdin
241 newsi = os.open(os.devnull, os.O_RDWR)
242 os.dup2(newsi, sys.stdin.fileno())
243
244 if umask:
245 os.umask(umask)
246
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500247 try:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600248 (realfn, virtual, mc) = bb.cache.virtualfn2realfn(fn)
249 the_data = databuilder.mcdata[mc]
250 the_data.setVar("BB_WORKERCONTEXT", "1")
251 the_data.setVar("BB_TASKDEPDATA", taskdepdata)
Andrew Geisslereff27472021-10-29 15:35:00 -0500252 the_data.setVar('BB_CURRENTTASK', taskname.replace("do_", ""))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500253 if cfg.limited_deps:
254 the_data.setVar("BB_LIMITEDDEPS", "1")
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600255 the_data.setVar("BUILDNAME", workerdata["buildname"])
256 the_data.setVar("DATE", workerdata["date"])
257 the_data.setVar("TIME", workerdata["time"])
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500258 for varname, value in extraconfigdata.items():
259 the_data.setVar(varname, value)
260
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600261 bb.parse.siggen.set_taskdata(workerdata["sigdata"])
Brad Bishop08902b02019-08-20 09:16:51 -0400262 if "newhashes" in workerdata:
263 bb.parse.siggen.set_taskhashes(workerdata["newhashes"])
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600264 ret = 0
265
Patrick Williamse760df82023-05-26 11:10:49 -0500266 the_data = databuilder.parseRecipe(fn, appends, layername)
Brad Bishop19323692019-04-05 15:28:33 -0400267 the_data.setVar('BB_TASKHASH', taskhash)
268 the_data.setVar('BB_UNIHASH', unihash)
Andrew Geissler517393d2023-01-13 08:55:19 -0600269 bb.parse.siggen.setup_datacache_from_datastore(fn, the_data)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500270
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500271 bb.utils.set_process_name("%s:%s" % (the_data.getVar("PN"), taskname.replace("do_", "")))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500272
Andrew Geissler6aa7eec2023-03-03 12:41:14 -0600273 if not bb.utils.to_boolean(the_data.getVarFlag(taskname, 'network')):
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000274 if bb.utils.is_local_uid(uid):
275 logger.debug("Attempting to disable network for %s" % taskname)
276 bb.utils.disable_network(uid, gid)
277 else:
278 logger.debug("Skipping disable network for %s since %s is not a local uid." % (taskname, uid))
Andrew Geissler595f6302022-01-24 19:11:47 +0000279
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500280 # exported_vars() returns a generator which *cannot* be passed to os.environ.update()
281 # successfully. We also need to unset anything from the environment which shouldn't be there
282 exports = bb.data.exported_vars(the_data)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600283
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500284 bb.utils.empty_environment()
285 for e, v in exports:
286 os.environ[e] = v
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600287
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500288 for e in fakeenv:
289 os.environ[e] = fakeenv[e]
290 the_data.setVar(e, fakeenv[e])
291 the_data.setVarFlag(e, 'export', "1")
292
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500293 task_exports = the_data.getVarFlag(taskname, 'exports')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600294 if task_exports:
295 for e in task_exports.split():
296 the_data.setVarFlag(e, 'export', '1')
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500297 v = the_data.getVar(e)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600298 if v is not None:
299 os.environ[e] = v
300
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500301 if quieterrors:
302 the_data.setVarFlag(taskname, "quieterrors", "1")
303
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500304 except Exception:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500305 if not quieterrors:
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500306 logger.critical(traceback.format_exc())
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500307 os._exit(1)
308 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500309 if dry_run:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500310 return 0
Andrew Geisslereff27472021-10-29 15:35:00 -0500311 try:
312 ret = bb.build.exec_task(fn, taskname, the_data, cfg.profile)
313 finally:
314 if fakeroot:
315 fakerootcmd = shlex.split(the_data.getVar("FAKEROOTCMD"))
316 subprocess.run(fakerootcmd + ['-S'], check=True, stdout=subprocess.PIPE)
Patrick Williams93c203f2021-10-06 16:15:23 -0500317 return ret
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500318 except:
319 os._exit(1)
320 if not profiling:
321 os._exit(child())
322 else:
323 profname = "profile-%s.log" % (fn.replace("/", "-") + "-" + taskname)
324 prof = profile.Profile()
325 try:
326 ret = profile.Profile.runcall(prof, child)
327 finally:
328 prof.dump_stats(profname)
329 bb.utils.process_profilelog(profname)
330 os._exit(ret)
331 else:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600332 for key, value in iter(envbackup.items()):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500333 if value is None:
334 del os.environ[key]
335 else:
336 os.environ[key] = value
337
338 return pid, pipein, pipeout
339
340class runQueueWorkerPipe():
341 """
342 Abstraction for a pipe between a worker thread and the worker server
343 """
344 def __init__(self, pipein, pipeout):
345 self.input = pipein
346 if pipeout:
347 pipeout.close()
348 bb.utils.nonblockingfd(self.input)
Andrew Geissler220dafd2023-10-04 10:18:08 -0500349 self.queue = bytearray()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500350
351 def read(self):
352 start = len(self.queue)
353 try:
Andrew Geissler220dafd2023-10-04 10:18:08 -0500354 self.queue.extend(self.input.read(102400) or b"")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500355 except (OSError, IOError) as e:
356 if e.errno != errno.EAGAIN:
357 raise
358
359 end = len(self.queue)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600360 index = self.queue.find(b"</event>")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500361 while index != -1:
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600362 msg = self.queue[:index+8]
363 assert msg.startswith(b"<event>") and msg.count(b"<event>") == 1
364 worker_fire_prepickled(msg)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500365 self.queue = self.queue[index+8:]
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600366 index = self.queue.find(b"</event>")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500367 return (end > start)
368
369 def close(self):
370 while self.read():
371 continue
372 if len(self.queue) > 0:
373 print("Warning, worker child left partial message: %s" % self.queue)
374 self.input.close()
375
376normalexit = False
377
378class BitbakeWorker(object):
379 def __init__(self, din):
380 self.input = din
381 bb.utils.nonblockingfd(self.input)
Andrew Geissler220dafd2023-10-04 10:18:08 -0500382 self.queue = bytearray()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500383 self.cookercfg = None
384 self.databuilder = None
385 self.data = None
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500386 self.extraconfigdata = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500387 self.build_pids = {}
388 self.build_pipes = {}
389
390 signal.signal(signal.SIGTERM, self.sigterm_exception)
391 # Let SIGHUP exit as SIGTERM
392 signal.signal(signal.SIGHUP, self.sigterm_exception)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500393 if "beef" in sys.argv[1]:
394 bb.utils.set_process_name("Worker (Fakeroot)")
395 else:
396 bb.utils.set_process_name("Worker")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500397
398 def sigterm_exception(self, signum, stackframe):
399 if signum == signal.SIGTERM:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500400 bb.warn("Worker received SIGTERM, shutting down...")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500401 elif signum == signal.SIGHUP:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500402 bb.warn("Worker received SIGHUP, shutting down...")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500403 self.handle_finishnow(None)
404 signal.signal(signal.SIGTERM, signal.SIG_DFL)
405 os.kill(os.getpid(), signal.SIGTERM)
406
407 def serve(self):
408 while True:
409 (ready, _, _) = select.select([self.input] + [i.input for i in self.build_pipes.values()], [] , [], 1)
410 if self.input in ready:
411 try:
412 r = self.input.read()
413 if len(r) == 0:
414 # EOF on pipe, server must have terminated
415 self.sigterm_exception(signal.SIGTERM, None)
Andrew Geissler220dafd2023-10-04 10:18:08 -0500416 self.queue.extend(r)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500417 except (OSError, IOError):
418 pass
419 if len(self.queue):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600420 self.handle_item(b"cookerconfig", self.handle_cookercfg)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500421 self.handle_item(b"extraconfigdata", self.handle_extraconfigdata)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600422 self.handle_item(b"workerdata", self.handle_workerdata)
Brad Bishop08902b02019-08-20 09:16:51 -0400423 self.handle_item(b"newtaskhashes", self.handle_newtaskhashes)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600424 self.handle_item(b"runtask", self.handle_runtask)
425 self.handle_item(b"finishnow", self.handle_finishnow)
426 self.handle_item(b"ping", self.handle_ping)
427 self.handle_item(b"quit", self.handle_quit)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500428
429 for pipe in self.build_pipes:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500430 if self.build_pipes[pipe].input in ready:
431 self.build_pipes[pipe].read()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500432 if len(self.build_pids):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500433 while self.process_waitpid():
434 continue
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500435
436
437 def handle_item(self, item, func):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600438 if self.queue.startswith(b"<" + item + b">"):
439 index = self.queue.find(b"</" + item + b">")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500440 while index != -1:
Andrew Geisslereff27472021-10-29 15:35:00 -0500441 try:
442 func(self.queue[(len(item) + 2):index])
443 except pickle.UnpicklingError:
444 workerlog_write("Unable to unpickle data: %s\n" % ":".join("{:02x}".format(c) for c in self.queue))
445 raise
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500446 self.queue = self.queue[(index + len(item) + 3):]
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600447 index = self.queue.find(b"</" + item + b">")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500448
449 def handle_cookercfg(self, data):
450 self.cookercfg = pickle.loads(data)
451 self.databuilder = bb.cookerdata.CookerDataBuilder(self.cookercfg, worker=True)
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000452 self.databuilder.parseBaseConfiguration(worker=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500453 self.data = self.databuilder.data
454
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500455 def handle_extraconfigdata(self, data):
456 self.extraconfigdata = pickle.loads(data)
457
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500458 def handle_workerdata(self, data):
459 self.workerdata = pickle.loads(data)
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500460 bb.build.verboseShellLogging = self.workerdata["build_verbose_shell"]
461 bb.build.verboseStdoutLogging = self.workerdata["build_verbose_stdout"]
Andrew Geissler82c905d2020-04-13 13:39:40 -0500462 bb.msg.loggerDefaultLogLevel = self.workerdata["logdefaultlevel"]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500463 bb.msg.loggerDefaultDomains = self.workerdata["logdefaultdomain"]
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600464 for mc in self.databuilder.mcdata:
465 self.databuilder.mcdata[mc].setVar("PRSERV_HOST", self.workerdata["prhost"])
Brad Bishopa34c0302019-09-23 22:34:48 -0400466 self.databuilder.mcdata[mc].setVar("BB_HASHSERVE", self.workerdata["hashservaddr"])
Patrick Williams92b42cb2022-09-03 06:53:57 -0500467 self.databuilder.mcdata[mc].setVar("__bbclasstype", "recipe")
Brad Bishop08902b02019-08-20 09:16:51 -0400468
469 def handle_newtaskhashes(self, data):
470 self.workerdata["newhashes"] = pickle.loads(data)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500471
472 def handle_ping(self, _):
473 workerlog_write("Handling ping\n")
474
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600475 logger.warning("Pong from bitbake-worker!")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500476
477 def handle_quit(self, data):
478 workerlog_write("Handling quit\n")
479
480 global normalexit
481 normalexit = True
482 sys.exit(0)
483
484 def handle_runtask(self, data):
Andrew Geissler517393d2023-01-13 08:55:19 -0600485 runtask = pickle.loads(data)
486
487 fn = runtask['fn']
488 task = runtask['task']
489 taskname = runtask['taskname']
490
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500491 workerlog_write("Handling runtask %s %s %s\n" % (task, fn, taskname))
492
Andrew Geissler517393d2023-01-13 08:55:19 -0600493 pid, pipein, pipeout = fork_off_task(self.cookercfg, self.data, self.databuilder, self.workerdata, self.extraconfigdata, runtask)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500494 self.build_pids[pid] = task
495 self.build_pipes[pid] = runQueueWorkerPipe(pipein, pipeout)
496
497 def process_waitpid(self):
498 """
499 Return none is there are no processes awaiting result collection, otherwise
500 collect the process exit codes and close the information pipe.
501 """
502 try:
503 pid, status = os.waitpid(-1, os.WNOHANG)
504 if pid == 0 or os.WIFSTOPPED(status):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500505 return False
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500506 except OSError:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500507 return False
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500508
509 workerlog_write("Exit code of %s for pid %s\n" % (status, pid))
510
511 if os.WIFEXITED(status):
512 status = os.WEXITSTATUS(status)
513 elif os.WIFSIGNALED(status):
514 # Per shell conventions for $?, when a process exits due to
515 # a signal, we return an exit code of 128 + SIGNUM
516 status = 128 + os.WTERMSIG(status)
517
518 task = self.build_pids[pid]
519 del self.build_pids[pid]
520
521 self.build_pipes[pid].close()
522 del self.build_pipes[pid]
523
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600524 worker_fire_prepickled(b"<exitcode>" + pickle.dumps((task, status)) + b"</exitcode>")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500525
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500526 return True
527
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500528 def handle_finishnow(self, _):
529 if self.build_pids:
530 logger.info("Sending SIGTERM to remaining %s tasks", len(self.build_pids))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600531 for k, v in iter(self.build_pids.items()):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500532 try:
533 os.kill(-k, signal.SIGTERM)
534 os.waitpid(-1, 0)
535 except:
536 pass
537 for pipe in self.build_pipes:
538 self.build_pipes[pipe].read()
539
540try:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600541 worker = BitbakeWorker(os.fdopen(sys.stdin.fileno(), 'rb'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500542 if not profiling:
543 worker.serve()
544 else:
545 profname = "profile-worker.log"
546 prof = profile.Profile()
547 try:
548 profile.Profile.runcall(prof, worker.serve)
549 finally:
550 prof.dump_stats(profname)
551 bb.utils.process_profilelog(profname)
552except BaseException as e:
553 if not normalexit:
554 import traceback
555 sys.stderr.write(traceback.format_exc())
556 sys.stderr.write(str(e))
Andrew Geissler5199d832021-09-24 16:47:35 -0500557finally:
558 worker_thread_exit = True
559 worker_thread.join()
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500560
Andrew Geisslerd159c7f2021-09-02 21:05:58 -0500561workerlog_write("exiting")
Andrew Geissler5199d832021-09-24 16:47:35 -0500562if not normalexit:
563 sys.exit(1)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500564sys.exit(0)