blob: 613956f30f3837de76029e3c5499de1869615b6b [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001#
2# BitBake Process based server.
3#
4# Copyright (C) 2010 Bob Foerster <robert@erafx.com>
5#
Brad Bishopc342db32019-05-15 21:57:59 -04006# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -05007#
Patrick Williamsc124f4f2015-09-15 14:41:29 -05008
9"""
10 This module implements a multiprocessing.Process based server for bitbake.
11"""
12
13import bb
14import bb.event
Patrick Williamsc124f4f2015-09-15 14:41:29 -050015import logging
16import multiprocessing
Brad Bishopd7bf8c12018-02-25 22:55:05 -050017import threading
18import array
Patrick Williamsc124f4f2015-09-15 14:41:29 -050019import os
Patrick Williamsc124f4f2015-09-15 14:41:29 -050020import sys
21import time
22import select
Brad Bishopd7bf8c12018-02-25 22:55:05 -050023import socket
24import subprocess
25import errno
26import re
27import datetime
Andrew Geisslerc9f78652020-09-18 14:11:35 -050028import pickle
Patrick Williams213cb262021-08-07 19:21:33 -050029import traceback
Patrick Williamsde0582f2022-04-08 10:23:27 -050030import gc
Brad Bishopd7bf8c12018-02-25 22:55:05 -050031import bb.server.xmlrpcserver
32from bb import daemonize
33from multiprocessing import queues
Patrick Williamsc124f4f2015-09-15 14:41:29 -050034
35logger = logging.getLogger('BitBake')
36
Brad Bishopd7bf8c12018-02-25 22:55:05 -050037class ProcessTimeout(SystemExit):
38 pass
Patrick Williamsc124f4f2015-09-15 14:41:29 -050039
Andrew Geisslerc9f78652020-09-18 14:11:35 -050040def serverlog(msg):
41 print(str(os.getpid()) + " " + datetime.datetime.now().strftime('%H:%M:%S.%f') + " " + msg)
42 sys.stdout.flush()
43
Andrew Geissler635e0e42020-08-21 15:58:33 -050044class ProcessServer():
Patrick Williamsc124f4f2015-09-15 14:41:29 -050045 profile_filename = "profile.log"
46 profile_processed_filename = "profile.log.processed"
47
Andrew Geisslerc9f78652020-09-18 14:11:35 -050048 def __init__(self, lock, lockname, sock, sockname, server_timeout, xmlrpcinterface):
Brad Bishopd7bf8c12018-02-25 22:55:05 -050049 self.command_channel = False
50 self.command_channel_reply = False
Patrick Williamsc124f4f2015-09-15 14:41:29 -050051 self.quit = False
Brad Bishop6e60e8b2018-02-01 10:27:11 -050052 self.heartbeat_seconds = 1 # default, BB_HEARTBEAT_EVENT will be checked once we have a datastore.
53 self.next_heartbeat = time.time()
Patrick Williamsc124f4f2015-09-15 14:41:29 -050054
Brad Bishopd7bf8c12018-02-25 22:55:05 -050055 self.event_handle = None
Andrew Geissler635e0e42020-08-21 15:58:33 -050056 self.hadanyui = False
Brad Bishopd7bf8c12018-02-25 22:55:05 -050057 self.haveui = False
Andrew Geisslerb7d28612020-07-24 16:15:54 -050058 self.maxuiwait = 30
Brad Bishopd7bf8c12018-02-25 22:55:05 -050059 self.xmlrpc = False
60
61 self._idlefuns = {}
62
63 self.bitbake_lock = lock
Andrew Geisslerc9f78652020-09-18 14:11:35 -050064 self.bitbake_lock_name = lockname
Brad Bishopd7bf8c12018-02-25 22:55:05 -050065 self.sock = sock
66 self.sockname = sockname
67
Andrew Geissler635e0e42020-08-21 15:58:33 -050068 self.server_timeout = server_timeout
Andrew Geisslerc9f78652020-09-18 14:11:35 -050069 self.timeout = self.server_timeout
Andrew Geissler635e0e42020-08-21 15:58:33 -050070 self.xmlrpcinterface = xmlrpcinterface
71
Brad Bishopd7bf8c12018-02-25 22:55:05 -050072 def register_idle_function(self, function, data):
73 """Register a function to be called while the server is idle"""
74 assert hasattr(function, '__call__')
75 self._idlefuns[function] = data
Patrick Williamsc124f4f2015-09-15 14:41:29 -050076
77 def run(self):
Brad Bishopd7bf8c12018-02-25 22:55:05 -050078
79 if self.xmlrpcinterface[0]:
80 self.xmlrpc = bb.server.xmlrpcserver.BitBakeXMLRPCServer(self.xmlrpcinterface, self.cooker, self)
81
Andrew Geisslerc9f78652020-09-18 14:11:35 -050082 serverlog("Bitbake XMLRPC server address: %s, server port: %s" % (self.xmlrpc.host, self.xmlrpc.port))
Brad Bishopd7bf8c12018-02-25 22:55:05 -050083
84 try:
85 self.bitbake_lock.seek(0)
86 self.bitbake_lock.truncate()
87 if self.xmlrpc:
88 self.bitbake_lock.write("%s %s:%s\n" % (os.getpid(), self.xmlrpc.host, self.xmlrpc.port))
89 else:
90 self.bitbake_lock.write("%s\n" % (os.getpid()))
91 self.bitbake_lock.flush()
92 except Exception as e:
Andrew Geisslerc9f78652020-09-18 14:11:35 -050093 serverlog("Error writing to lock file: %s" % str(e))
Brad Bishopd7bf8c12018-02-25 22:55:05 -050094 pass
95
96 if self.cooker.configuration.profile:
97 try:
98 import cProfile as profile
99 except:
100 import profile
101 prof = profile.Profile()
102
103 ret = profile.Profile.runcall(prof, self.main)
104
105 prof.dump_stats("profile.log")
106 bb.utils.process_profilelog("profile.log")
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500107 serverlog("Raw profiling information saved to profile.log and processed statistics to profile.log.processed")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500108
109 else:
110 ret = self.main()
111
112 return ret
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500113
114 def main(self):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500115 self.cooker.pre_serve()
116
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500117 bb.utils.set_process_name("Cooker")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500118
119 ready = []
Brad Bishopf058f492019-01-28 23:50:33 -0500120 newconnections = []
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500121
122 self.controllersock = False
123 fds = [self.sock]
124 if self.xmlrpc:
125 fds.append(self.xmlrpc)
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500126 seendata = False
127 serverlog("Entering server connection loop")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500128
129 def disconnect_client(self, fds):
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500130 serverlog("Disconnecting Client")
Brad Bishopf058f492019-01-28 23:50:33 -0500131 if self.controllersock:
132 fds.remove(self.controllersock)
133 self.controllersock.close()
134 self.controllersock = False
135 if self.haveui:
136 fds.remove(self.command_channel)
137 bb.event.unregister_UIHhandler(self.event_handle, True)
138 self.command_channel_reply.writer.close()
139 self.event_writer.writer.close()
140 self.command_channel.close()
141 self.command_channel = False
142 del self.event_writer
143 self.lastui = time.time()
144 self.cooker.clientComplete()
145 self.haveui = False
146 ready = select.select(fds,[],[],0)[0]
147 if newconnections:
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500148 serverlog("Starting new client")
Brad Bishopf058f492019-01-28 23:50:33 -0500149 conn = newconnections.pop(-1)
150 fds.append(conn)
151 self.controllersock = conn
Andrew Geissler5f350902021-07-23 13:09:54 -0400152 elif not self.timeout and not ready:
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500153 serverlog("No timeout, exiting.")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500154 self.quit = True
155
Andrew Geisslerb7d28612020-07-24 16:15:54 -0500156 self.lastui = time.time()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500157 while not self.quit:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500158 if self.sock in ready:
Brad Bishopf058f492019-01-28 23:50:33 -0500159 while select.select([self.sock],[],[],0)[0]:
160 controllersock, address = self.sock.accept()
161 if self.controllersock:
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500162 serverlog("Queuing %s (%s)" % (str(ready), str(newconnections)))
Brad Bishopf058f492019-01-28 23:50:33 -0500163 newconnections.append(controllersock)
164 else:
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500165 serverlog("Accepting %s (%s)" % (str(ready), str(newconnections)))
Brad Bishopf058f492019-01-28 23:50:33 -0500166 self.controllersock = controllersock
167 fds.append(controllersock)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500168 if self.controllersock in ready:
169 try:
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500170 serverlog("Processing Client")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500171 ui_fds = recvfds(self.controllersock, 3)
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500172 serverlog("Connecting Client")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500173
174 # Where to write events to
175 writer = ConnectionWriter(ui_fds[0])
176 self.event_handle = bb.event.register_UIHhandler(writer, True)
177 self.event_writer = writer
178
179 # Where to read commands from
180 reader = ConnectionReader(ui_fds[1])
181 fds.append(reader)
182 self.command_channel = reader
183
184 # Where to send command return values to
185 writer = ConnectionWriter(ui_fds[2])
186 self.command_channel_reply = writer
187
188 self.haveui = True
Andrew Geissler635e0e42020-08-21 15:58:33 -0500189 self.hadanyui = True
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500190
191 except (EOFError, OSError):
192 disconnect_client(self, fds)
193
Andrew Geisslerb7d28612020-07-24 16:15:54 -0500194 if not self.timeout == -1.0 and not self.haveui and self.timeout and \
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500195 (self.lastui + self.timeout) < time.time():
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500196 serverlog("Server timeout, exiting.")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500197 self.quit = True
198
Andrew Geisslerb7d28612020-07-24 16:15:54 -0500199 # If we don't see a UI connection within maxuiwait, its unlikely we're going to see
200 # one. We have had issue with processes hanging indefinitely so timing out UI-less
201 # servers is useful.
Andrew Geissler635e0e42020-08-21 15:58:33 -0500202 if not self.hadanyui and not self.xmlrpc and not self.timeout and (self.lastui + self.maxuiwait) < time.time():
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500203 serverlog("No UI connection within max timeout, exiting to avoid infinite loop.")
Andrew Geisslerb7d28612020-07-24 16:15:54 -0500204 self.quit = True
205
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500206 if self.command_channel in ready:
207 try:
208 command = self.command_channel.get()
209 except EOFError:
210 # Client connection shutting down
211 ready = []
212 disconnect_client(self, fds)
213 continue
214 if command[0] == "terminateServer":
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500215 self.quit = True
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500216 continue
217 try:
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500218 serverlog("Running command %s" % command)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500219 self.command_channel_reply.send(self.cooker.command.runCommand(command))
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500220 serverlog("Command Completed")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500221 except Exception as e:
Patrick Williams213cb262021-08-07 19:21:33 -0500222 stack = traceback.format_exc()
223 serverlog('Exception in server main event loop running command %s (%s)' % (command, stack))
224 logger.exception('Exception in server main event loop running command %s (%s)' % (command, stack))
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500225
226 if self.xmlrpc in ready:
227 self.xmlrpc.handle_requests()
228
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500229 if not seendata and hasattr(self.cooker, "data"):
230 heartbeat_event = self.cooker.data.getVar('BB_HEARTBEAT_EVENT')
231 if heartbeat_event:
232 try:
233 self.heartbeat_seconds = float(heartbeat_event)
234 except:
235 bb.warn('Ignoring invalid BB_HEARTBEAT_EVENT=%s, must be a float specifying seconds.' % heartbeat_event)
236
237 self.timeout = self.server_timeout or self.cooker.data.getVar('BB_SERVER_TIMEOUT')
238 try:
239 if self.timeout:
240 self.timeout = float(self.timeout)
241 except:
242 bb.warn('Ignoring invalid BB_SERVER_TIMEOUT=%s, must be a float specifying seconds.' % self.timeout)
243 seendata = True
244
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500245 ready = self.idle_commands(.1, fds)
246
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500247 serverlog("Exiting")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500248 # Remove the socket file so we don't get any more connections to avoid races
Andrew Geisslerb7d28612020-07-24 16:15:54 -0500249 try:
250 os.unlink(self.sockname)
251 except:
252 pass
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500253 self.sock.close()
254
Andrew Geisslerb7d28612020-07-24 16:15:54 -0500255 try:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500256 self.cooker.shutdown(True)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400257 self.cooker.notifier.stop()
258 self.cooker.confignotifier.stop()
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500259 except:
260 pass
261
262 self.cooker.post_serve()
263
Andrew Geissler9aee5002022-03-30 16:27:02 +0000264 if len(threading.enumerate()) != 1:
265 serverlog("More than one thread left?: " + str(threading.enumerate()))
266
Andrew Geissler635e0e42020-08-21 15:58:33 -0500267 # Flush logs before we release the lock
268 sys.stdout.flush()
269 sys.stderr.flush()
270
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500271 # Finally release the lockfile but warn about other processes holding it open
272 lock = self.bitbake_lock
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500273 lockfile = self.bitbake_lock_name
274
275 def get_lock_contents(lockfile):
276 try:
277 with open(lockfile, "r") as f:
278 return f.readlines()
279 except FileNotFoundError:
280 return None
281
282 lockcontents = get_lock_contents(lockfile)
283 serverlog("Original lockfile contents: " + str(lockcontents))
284
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500285 lock.close()
286 lock = None
287
288 while not lock:
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500289 i = 0
290 lock = None
291 while not lock and i < 30:
292 lock = bb.utils.lockfile(lockfile, shared=False, retry=False, block=False)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500293 if not lock:
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500294 newlockcontents = get_lock_contents(lockfile)
295 if newlockcontents != lockcontents:
296 # A new server was started, the lockfile contents changed, we can exit
297 serverlog("Lockfile now contains different contents, exiting: " + str(newlockcontents))
298 return
299 time.sleep(0.1)
300 i += 1
301 if lock:
302 # We hold the lock so we can remove the file (hide stale pid data)
303 # via unlockfile.
304 bb.utils.unlockfile(lock)
305 serverlog("Exiting as we could obtain the lock")
306 return
307
308 if not lock:
309 # Some systems may not have lsof available
310 procs = None
311 try:
312 procs = subprocess.check_output(["lsof", '-w', lockfile], stderr=subprocess.STDOUT)
313 except subprocess.CalledProcessError:
314 # File was deleted?
315 continue
316 except OSError as e:
317 if e.errno != errno.ENOENT:
318 raise
319 if procs is None:
320 # Fall back to fuser if lsof is unavailable
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500321 try:
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500322 procs = subprocess.check_output(["fuser", '-v', lockfile], stderr=subprocess.STDOUT)
323 except subprocess.CalledProcessError:
324 # File was deleted?
325 continue
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500326 except OSError as e:
327 if e.errno != errno.ENOENT:
328 raise
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500329
Andrew Geissler595f6302022-01-24 19:11:47 +0000330 msg = ["Delaying shutdown due to active processes which appear to be holding bitbake.lock"]
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500331 if procs:
Andrew Geissler595f6302022-01-24 19:11:47 +0000332 msg.append(":\n%s" % str(procs.decode("utf-8")))
333 serverlog("".join(msg))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500334
335 def idle_commands(self, delay, fds=None):
336 nextsleep = delay
337 if not fds:
338 fds = []
339
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600340 for function, data in list(self._idlefuns.items()):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500341 try:
342 retval = function(self, data, False)
343 if retval is False:
344 del self._idlefuns[function]
345 nextsleep = None
346 elif retval is True:
347 nextsleep = None
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600348 elif isinstance(retval, float) and nextsleep:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500349 if (retval < nextsleep):
350 nextsleep = retval
351 elif nextsleep is None:
352 continue
353 else:
354 fds = fds + retval
355 except SystemExit:
356 raise
357 except Exception as exc:
358 if not isinstance(exc, bb.BBHandledException):
359 logger.exception('Running idle function')
360 del self._idlefuns[function]
361 self.quit = True
362
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500363 # Create new heartbeat event?
364 now = time.time()
365 if now >= self.next_heartbeat:
366 # We might have missed heartbeats. Just trigger once in
367 # that case and continue after the usual delay.
368 self.next_heartbeat += self.heartbeat_seconds
369 if self.next_heartbeat <= now:
370 self.next_heartbeat = now + self.heartbeat_seconds
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500371 if hasattr(self.cooker, "data"):
372 heartbeat = bb.event.HeartbeatEvent(now)
William A. Kennington IIIac69b482021-06-02 12:28:27 -0700373 try:
374 bb.event.fire(heartbeat, self.cooker.data)
375 except Exception as exc:
376 if not isinstance(exc, bb.BBHandledException):
377 logger.exception('Running heartbeat function')
378 self.quit = True
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500379 if nextsleep and now + nextsleep > self.next_heartbeat:
380 # Shorten timeout so that we we wake up in time for
381 # the heartbeat.
382 nextsleep = self.next_heartbeat - now
383
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500384 if nextsleep is not None:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500385 if self.xmlrpc:
386 nextsleep = self.xmlrpc.get_timeout(nextsleep)
387 try:
388 return select.select(fds,[],[],nextsleep)[0]
389 except InterruptedError:
390 # Ignore EINTR
391 return []
392 else:
393 return select.select(fds,[],[],0)[0]
394
395
396class ServerCommunicator():
397 def __init__(self, connection, recv):
398 self.connection = connection
399 self.recv = recv
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500400
401 def runCommand(self, command):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500402 self.connection.send(command)
403 if not self.recv.poll(30):
Andrew Geisslerb7d28612020-07-24 16:15:54 -0500404 logger.info("No reply from server in 30s")
Andrew Geissler475cb722020-07-10 16:00:51 -0500405 if not self.recv.poll(30):
406 raise ProcessTimeout("Timeout while waiting for a reply from the bitbake server (60s)")
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500407 ret, exc = self.recv.get()
408 # Should probably turn all exceptions in exc back into exceptions?
409 # For now, at least handle BBHandledException
410 if exc and ("BBHandledException" in exc or "SystemExit" in exc):
411 raise bb.BBHandledException()
412 return ret, exc
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500413
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500414 def updateFeatureSet(self, featureset):
415 _, error = self.runCommand(["setFeatures", featureset])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500416 if error:
417 logger.error("Unable to set the cooker to the correct featureset: %s" % error)
418 raise BaseException(error)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500419
420 def getEventHandle(self):
421 handle, error = self.runCommand(["getUIHandlerNum"])
422 if error:
423 logger.error("Unable to get UI Handler Number: %s" % error)
424 raise BaseException(error)
425
426 return handle
427
428 def terminateServer(self):
429 self.connection.send(['terminateServer'])
430 return
431
432class BitBakeProcessServerConnection(object):
433 def __init__(self, ui_channel, recv, eq, sock):
434 self.connection = ServerCommunicator(ui_channel, recv)
435 self.events = eq
436 # Save sock so it doesn't get gc'd for the life of our connection
437 self.socket_connection = sock
438
439 def terminate(self):
440 self.socket_connection.close()
441 self.connection.connection.close()
442 self.connection.recv.close()
443 return
444
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500445start_log_format = '--- Starting bitbake server pid %s at %s ---'
446start_log_datetime_format = '%Y-%m-%d %H:%M:%S.%f'
447
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500448class BitBakeServer(object):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500449
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500450 def __init__(self, lock, sockname, featureset, server_timeout, xmlrpcinterface):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500451
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500452 self.server_timeout = server_timeout
453 self.xmlrpcinterface = xmlrpcinterface
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500454 self.featureset = featureset
455 self.sockname = sockname
456 self.bitbake_lock = lock
457 self.readypipe, self.readypipein = os.pipe()
458
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800459 # Place the log in the builddirectory alongside the lock file
460 logfile = os.path.join(os.path.dirname(self.bitbake_lock.name), "bitbake-cookerdaemon.log")
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500461 self.logfile = logfile
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800462
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500463 startdatetime = datetime.datetime.now()
464 bb.daemonize.createDaemon(self._startServer, logfile)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500465 self.bitbake_lock.close()
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800466 os.close(self.readypipein)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500467
468 ready = ConnectionReader(self.readypipe)
Brad Bishopf058f492019-01-28 23:50:33 -0500469 r = ready.poll(5)
470 if not r:
471 bb.note("Bitbake server didn't start within 5 seconds, waiting for 90")
472 r = ready.poll(90)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500473 if r:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800474 try:
475 r = ready.get()
476 except EOFError:
Andrew Geisslerd159c7f2021-09-02 21:05:58 -0500477 # Trap the child exiting/closing the pipe and error out
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800478 r = None
Brad Bishopf058f492019-01-28 23:50:33 -0500479 if not r or r[0] != "r":
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500480 ready.close()
Brad Bishopf058f492019-01-28 23:50:33 -0500481 bb.error("Unable to start bitbake server (%s)" % str(r))
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500482 if os.path.exists(logfile):
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500483 logstart_re = re.compile(start_log_format % ('([0-9]+)', '([0-9-]+ [0-9:.]+)'))
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500484 started = False
485 lines = []
Brad Bishopa5c52ff2018-11-23 10:55:50 +1300486 lastlines = []
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500487 with open(logfile, "r") as f:
488 for line in f:
489 if started:
490 lines.append(line)
491 else:
Brad Bishopa5c52ff2018-11-23 10:55:50 +1300492 lastlines.append(line)
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500493 res = logstart_re.search(line.rstrip())
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500494 if res:
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500495 ldatetime = datetime.datetime.strptime(res.group(2), start_log_datetime_format)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500496 if ldatetime >= startdatetime:
497 started = True
498 lines.append(line)
Brad Bishopa5c52ff2018-11-23 10:55:50 +1300499 if len(lastlines) > 60:
500 lastlines = lastlines[-60:]
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500501 if lines:
Brad Bishopa5c52ff2018-11-23 10:55:50 +1300502 if len(lines) > 60:
503 bb.error("Last 60 lines of server log for this session (%s):\n%s" % (logfile, "".join(lines[-60:])))
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500504 else:
505 bb.error("Server log for this session (%s):\n%s" % (logfile, "".join(lines)))
Brad Bishopa5c52ff2018-11-23 10:55:50 +1300506 elif lastlines:
507 bb.error("Server didn't start, last 60 loglines (%s):\n%s" % (logfile, "".join(lastlines)))
508 else:
509 bb.error("%s doesn't exist" % logfile)
510
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500511 raise SystemExit(1)
Brad Bishopa5c52ff2018-11-23 10:55:50 +1300512
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500513 ready.close()
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500514
515 def _startServer(self):
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500516 os.close(self.readypipe)
517 os.set_inheritable(self.bitbake_lock.fileno(), True)
518 os.set_inheritable(self.readypipein, True)
519 serverscript = os.path.realpath(os.path.dirname(__file__) + "/../../../bin/bitbake-server")
Andrew Geisslerc926e172021-05-07 16:11:35 -0500520 os.execl(sys.executable, "bitbake-server", serverscript, "decafbad", str(self.bitbake_lock.fileno()), str(self.readypipein), self.logfile, self.bitbake_lock.name, self.sockname, str(self.server_timeout or 0), str(self.xmlrpcinterface[0]), str(self.xmlrpcinterface[1]))
Brad Bishope2d5b612018-11-23 10:55:50 +1300521
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500522def execServer(lockfd, readypipeinfd, lockname, sockname, server_timeout, xmlrpcinterface):
523
524 import bb.cookerdata
525 import bb.cooker
526
527 serverlog(start_log_format % (os.getpid(), datetime.datetime.now().strftime(start_log_datetime_format)))
528
529 try:
530 bitbake_lock = os.fdopen(lockfd, "w")
531
532 # Create server control socket
533 if os.path.exists(sockname):
534 os.unlink(sockname)
535
536 sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
537 # AF_UNIX has path length issues so chdir here to workaround
538 cwd = os.getcwd()
Brad Bishop08902b02019-08-20 09:16:51 -0400539 try:
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500540 os.chdir(os.path.dirname(sockname))
541 sock.bind(os.path.basename(sockname))
Andrew Geissler635e0e42020-08-21 15:58:33 -0500542 finally:
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500543 os.chdir(cwd)
544 sock.listen(1)
545
546 server = ProcessServer(bitbake_lock, lockname, sock, sockname, server_timeout, xmlrpcinterface)
547 writer = ConnectionWriter(readypipeinfd)
548 try:
549 featureset = []
550 cooker = bb.cooker.BBCooker(featureset, server.register_idle_function)
551 except bb.BBHandledException:
552 return None
553 writer.send("r")
554 writer.close()
555 server.cooker = cooker
556 serverlog("Started bitbake server pid %d" % os.getpid())
557
558 server.run()
559 finally:
Andrew Geissler9aee5002022-03-30 16:27:02 +0000560 # Flush any messages/errors to the logfile before exit
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500561 sys.stdout.flush()
562 sys.stderr.flush()
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500563
564def connectProcessServer(sockname, featureset):
565 # Connect to socket
566 sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
567 # AF_UNIX has path length issues so chdir here to workaround
568 cwd = os.getcwd()
569
Brad Bishopf058f492019-01-28 23:50:33 -0500570 readfd = writefd = readfd1 = writefd1 = readfd2 = writefd2 = None
571 eq = command_chan_recv = command_chan = None
572
573 sock.settimeout(10)
574
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500575 try:
Brad Bishope2d5b612018-11-23 10:55:50 +1300576 try:
577 os.chdir(os.path.dirname(sockname))
Brad Bishopf058f492019-01-28 23:50:33 -0500578 finished = False
579 while not finished:
580 try:
581 sock.connect(os.path.basename(sockname))
582 finished = True
583 except IOError as e:
584 if e.errno == errno.EWOULDBLOCK:
585 pass
Richard Purdie3da11142019-02-05 21:34:37 +0000586 raise
Brad Bishope2d5b612018-11-23 10:55:50 +1300587 finally:
588 os.chdir(cwd)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500589
590 # Send an fd for the remote to write events to
591 readfd, writefd = os.pipe()
592 eq = BBUIEventQueue(readfd)
593 # Send an fd for the remote to recieve commands from
594 readfd1, writefd1 = os.pipe()
595 command_chan = ConnectionWriter(writefd1)
596 # Send an fd for the remote to write commands results to
597 readfd2, writefd2 = os.pipe()
598 command_chan_recv = ConnectionReader(readfd2)
599
600 sendfds(sock, [writefd, readfd1, writefd2])
601
602 server_connection = BitBakeProcessServerConnection(command_chan, command_chan_recv, eq, sock)
603
604 # Close the ends of the pipes we won't use
605 for i in [writefd, readfd1, writefd2]:
606 os.close(i)
607
608 server_connection.connection.updateFeatureSet(featureset)
609
610 except (Exception, SystemExit) as e:
611 if command_chan_recv:
612 command_chan_recv.close()
613 if command_chan:
614 command_chan.close()
615 for i in [writefd, readfd1, writefd2]:
616 try:
Brad Bishope2d5b612018-11-23 10:55:50 +1300617 if i:
618 os.close(i)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500619 except OSError:
620 pass
621 sock.close()
622 raise
623
624 return server_connection
625
626def sendfds(sock, fds):
627 '''Send an array of fds over an AF_UNIX socket.'''
628 fds = array.array('i', fds)
629 msg = bytes([len(fds) % 256])
630 sock.sendmsg([msg], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, fds)])
631
632def recvfds(sock, size):
633 '''Receive an array of fds over an AF_UNIX socket.'''
634 a = array.array('i')
635 bytes_size = a.itemsize * size
636 msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size))
637 if not msg and not ancdata:
638 raise EOFError
639 try:
640 if len(ancdata) != 1:
641 raise RuntimeError('received %d items of ancdata' %
642 len(ancdata))
643 cmsg_level, cmsg_type, cmsg_data = ancdata[0]
644 if (cmsg_level == socket.SOL_SOCKET and
645 cmsg_type == socket.SCM_RIGHTS):
646 if len(cmsg_data) % a.itemsize != 0:
647 raise ValueError
648 a.frombytes(cmsg_data)
649 assert len(a) % 256 == msg[0]
650 return list(a)
651 except (ValueError, IndexError):
652 pass
653 raise RuntimeError('Invalid data received')
654
655class BBUIEventQueue:
656 def __init__(self, readfd):
657
658 self.eventQueue = []
659 self.eventQueueLock = threading.Lock()
660 self.eventQueueNotify = threading.Event()
661
662 self.reader = ConnectionReader(readfd)
663
664 self.t = threading.Thread()
Andrew Geissler5199d832021-09-24 16:47:35 -0500665 self.t.daemon = True
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500666 self.t.run = self.startCallbackHandler
667 self.t.start()
668
669 def getEvent(self):
670 self.eventQueueLock.acquire()
671
672 if len(self.eventQueue) == 0:
673 self.eventQueueLock.release()
674 return None
675
676 item = self.eventQueue.pop(0)
677
678 if len(self.eventQueue) == 0:
679 self.eventQueueNotify.clear()
680
681 self.eventQueueLock.release()
682 return item
683
684 def waitEvent(self, delay):
685 self.eventQueueNotify.wait(delay)
686 return self.getEvent()
687
688 def queue_event(self, event):
689 self.eventQueueLock.acquire()
690 self.eventQueue.append(event)
691 self.eventQueueNotify.set()
692 self.eventQueueLock.release()
693
694 def send_event(self, event):
695 self.queue_event(pickle.loads(event))
696
697 def startCallbackHandler(self):
698 bb.utils.set_process_name("UIEventQueue")
699 while True:
700 try:
701 self.reader.wait()
702 event = self.reader.get()
703 self.queue_event(event)
704 except EOFError:
705 # Easiest way to exit is to close the file descriptor to cause an exit
706 break
707 self.reader.close()
708
709class ConnectionReader(object):
710
711 def __init__(self, fd):
712 self.reader = multiprocessing.connection.Connection(fd, writable=False)
713 self.rlock = multiprocessing.Lock()
714
715 def wait(self, timeout=None):
716 return multiprocessing.connection.wait([self.reader], timeout)
717
718 def poll(self, timeout=None):
719 return self.reader.poll(timeout)
720
721 def get(self):
722 with self.rlock:
723 res = self.reader.recv_bytes()
724 return multiprocessing.reduction.ForkingPickler.loads(res)
725
726 def fileno(self):
727 return self.reader.fileno()
728
729 def close(self):
730 return self.reader.close()
731
732
733class ConnectionWriter(object):
734
735 def __init__(self, fd):
736 self.writer = multiprocessing.connection.Connection(fd, readable=False)
737 self.wlock = multiprocessing.Lock()
738 # Why bb.event needs this I have no idea
739 self.event = self
740
Andrew Geissler9aee5002022-03-30 16:27:02 +0000741 def _send(self, obj):
Patrick Williamsde0582f2022-04-08 10:23:27 -0500742 gc.disable()
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500743 with self.wlock:
744 self.writer.send_bytes(obj)
Patrick Williamsde0582f2022-04-08 10:23:27 -0500745 gc.enable()
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500746
Andrew Geissler9aee5002022-03-30 16:27:02 +0000747 def send(self, obj):
748 obj = multiprocessing.reduction.ForkingPickler.dumps(obj)
749 # See notes/code in CookerParser
750 # We must not terminate holding this lock else processes will hang.
751 # For SIGTERM, raising afterwards avoids this.
752 # For SIGINT, we don't want to have written partial data to the pipe.
753 # pthread_sigmask block/unblock would be nice but doesn't work, https://bugs.python.org/issue47139
754 process = multiprocessing.current_process()
755 if process and hasattr(process, "queue_signals"):
756 with process.signal_threadlock:
757 process.queue_signals = True
758 self._send(obj)
759 process.queue_signals = False
760 for sig in process.signal_received.pop():
761 process.handle_sig(sig, None)
762 else:
763 self._send(obj)
764
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500765 def fileno(self):
766 return self.writer.fileno()
767
768 def close(self):
769 return self.writer.close()