blob: db417c84280d5145dbaf66e743dfb094fd14298f [file] [log] [blame]
#
# BitBake Process based server.
#
# Copyright (C) 2010 Bob Foerster <robert@erafx.com>
#
# SPDX-License-Identifier: GPL-2.0-only
#
"""
This module implements a multiprocessing.Process based server for bitbake.
"""
import bb
import bb.event
import logging
import multiprocessing
import threading
import array
import os
import sys
import time
import select
import socket
import subprocess
import errno
import re
import datetime
import pickle
import traceback
import gc
import stat
import bb.server.xmlrpcserver
from bb import daemonize
from multiprocessing import queues
logger = logging.getLogger('BitBake')
class ProcessTimeout(SystemExit):
pass
def serverlog(msg):
print(str(os.getpid()) + " " + datetime.datetime.now().strftime('%H:%M:%S.%f') + " " + msg)
sys.stdout.flush()
#
# When we have lockfile issues, try and find infomation about which process is
# using the lockfile
#
def get_lockfile_process_msg(lockfile):
# Some systems may not have lsof available
procs = None
try:
procs = subprocess.check_output(["lsof", '-w', lockfile], stderr=subprocess.STDOUT)
except subprocess.CalledProcessError:
# File was deleted?
pass
except OSError as e:
if e.errno != errno.ENOENT:
raise
if procs is None:
# Fall back to fuser if lsof is unavailable
try:
procs = subprocess.check_output(["fuser", '-v', lockfile], stderr=subprocess.STDOUT)
except subprocess.CalledProcessError:
# File was deleted?
pass
except OSError as e:
if e.errno != errno.ENOENT:
raise
if procs:
return procs.decode("utf-8")
return None
class idleFinish():
def __init__(self, msg):
self.msg = msg
class ProcessServer():
profile_filename = "profile.log"
profile_processed_filename = "profile.log.processed"
def __init__(self, lock, lockname, sock, sockname, server_timeout, xmlrpcinterface):
self.command_channel = False
self.command_channel_reply = False
self.quit = False
self.heartbeat_seconds = 1 # default, BB_HEARTBEAT_EVENT will be checked once we have a datastore.
self.next_heartbeat = time.time()
self.event_handle = None
self.hadanyui = False
self.haveui = False
self.maxuiwait = 30
self.xmlrpc = False
self.idle = None
# Need a lock for _idlefuns changes
self._idlefuns = {}
self._idlefuncsLock = threading.Lock()
self.idle_cond = threading.Condition(self._idlefuncsLock)
self.bitbake_lock = lock
self.bitbake_lock_name = lockname
self.sock = sock
self.sockname = sockname
# It is possible the directory may be renamed. Cache the inode of the socket file
# so we can tell if things changed.
self.sockinode = os.stat(self.sockname)[stat.ST_INO]
self.server_timeout = server_timeout
self.timeout = self.server_timeout
self.xmlrpcinterface = xmlrpcinterface
def register_idle_function(self, function, data):
"""Register a function to be called while the server is idle"""
assert hasattr(function, '__call__')
with bb.utils.lock_timeout(self._idlefuncsLock):
self._idlefuns[function] = data
serverlog("Registering idle function %s" % str(function))
def run(self):
if self.xmlrpcinterface[0]:
self.xmlrpc = bb.server.xmlrpcserver.BitBakeXMLRPCServer(self.xmlrpcinterface, self.cooker, self)
serverlog("Bitbake XMLRPC server address: %s, server port: %s" % (self.xmlrpc.host, self.xmlrpc.port))
try:
self.bitbake_lock.seek(0)
self.bitbake_lock.truncate()
if self.xmlrpc:
self.bitbake_lock.write("%s %s:%s\n" % (os.getpid(), self.xmlrpc.host, self.xmlrpc.port))
else:
self.bitbake_lock.write("%s\n" % (os.getpid()))
self.bitbake_lock.flush()
except Exception as e:
serverlog("Error writing to lock file: %s" % str(e))
pass
if self.cooker.configuration.profile:
try:
import cProfile as profile
except:
import profile
prof = profile.Profile()
ret = profile.Profile.runcall(prof, self.main)
prof.dump_stats("profile.log")
bb.utils.process_profilelog("profile.log")
serverlog("Raw profiling information saved to profile.log and processed statistics to profile.log.processed")
else:
ret = self.main()
return ret
def _idle_check(self):
return len(self._idlefuns) == 0 and self.cooker.command.currentAsyncCommand is None
def wait_for_idle(self, timeout=30):
# Wait for the idle loop to have cleared
with bb.utils.lock_timeout(self._idlefuncsLock):
return self.idle_cond.wait_for(self._idle_check, timeout) is not False
def set_async_cmd(self, cmd):
with bb.utils.lock_timeout(self._idlefuncsLock):
ret = self.idle_cond.wait_for(self._idle_check, 30)
if ret is False:
return False
self.cooker.command.currentAsyncCommand = cmd
return True
def clear_async_cmd(self):
with bb.utils.lock_timeout(self._idlefuncsLock):
self.cooker.command.currentAsyncCommand = None
self.idle_cond.notify_all()
def get_async_cmd(self):
with bb.utils.lock_timeout(self._idlefuncsLock):
return self.cooker.command.currentAsyncCommand
def main(self):
self.cooker.pre_serve()
bb.utils.set_process_name("Cooker")
ready = []
newconnections = []
self.controllersock = False
fds = [self.sock]
if self.xmlrpc:
fds.append(self.xmlrpc)
seendata = False
serverlog("Entering server connection loop")
serverlog("Lockfile is: %s\nSocket is %s (%s)" % (self.bitbake_lock_name, self.sockname, os.path.exists(self.sockname)))
def disconnect_client(self, fds):
serverlog("Disconnecting Client (socket: %s)" % os.path.exists(self.sockname))
if self.controllersock:
fds.remove(self.controllersock)
self.controllersock.close()
self.controllersock = False
if self.haveui:
# Wait for the idle loop to have cleared (30s max)
if not self.wait_for_idle(30):
serverlog("Idle loop didn't finish queued commands after 30s, exiting.")
self.quit = True
fds.remove(self.command_channel)
bb.event.unregister_UIHhandler(self.event_handle, True)
self.command_channel_reply.writer.close()
self.event_writer.writer.close()
self.command_channel.close()
self.command_channel = False
del self.event_writer
self.lastui = time.time()
self.cooker.clientComplete()
self.haveui = False
ready = select.select(fds,[],[],0)[0]
if newconnections and not self.quit:
serverlog("Starting new client")
conn = newconnections.pop(-1)
fds.append(conn)
self.controllersock = conn
elif not self.timeout and not ready:
serverlog("No timeout, exiting.")
self.quit = True
self.lastui = time.time()
while not self.quit:
if self.sock in ready:
while select.select([self.sock],[],[],0)[0]:
controllersock, address = self.sock.accept()
if self.controllersock:
serverlog("Queuing %s (%s)" % (str(ready), str(newconnections)))
newconnections.append(controllersock)
else:
serverlog("Accepting %s (%s)" % (str(ready), str(newconnections)))
self.controllersock = controllersock
fds.append(controllersock)
if self.controllersock in ready:
try:
serverlog("Processing Client")
ui_fds = recvfds(self.controllersock, 3)
serverlog("Connecting Client")
# Where to write events to
writer = ConnectionWriter(ui_fds[0])
self.event_handle = bb.event.register_UIHhandler(writer, True)
self.event_writer = writer
# Where to read commands from
reader = ConnectionReader(ui_fds[1])
fds.append(reader)
self.command_channel = reader
# Where to send command return values to
writer = ConnectionWriter(ui_fds[2])
self.command_channel_reply = writer
self.haveui = True
self.hadanyui = True
except (EOFError, OSError):
disconnect_client(self, fds)
if not self.timeout == -1.0 and not self.haveui and self.timeout and \
(self.lastui + self.timeout) < time.time():
serverlog("Server timeout, exiting.")
self.quit = True
# If we don't see a UI connection within maxuiwait, its unlikely we're going to see
# one. We have had issue with processes hanging indefinitely so timing out UI-less
# servers is useful.
if not self.hadanyui and not self.xmlrpc and not self.timeout and (self.lastui + self.maxuiwait) < time.time():
serverlog("No UI connection within max timeout, exiting to avoid infinite loop.")
self.quit = True
if self.command_channel in ready:
try:
command = self.command_channel.get()
except EOFError:
# Client connection shutting down
ready = []
disconnect_client(self, fds)
continue
if command[0] == "terminateServer":
self.quit = True
continue
try:
serverlog("Running command %s" % command)
self.command_channel_reply.send(self.cooker.command.runCommand(command, self))
serverlog("Command Completed (socket: %s)" % os.path.exists(self.sockname))
except Exception as e:
stack = traceback.format_exc()
serverlog('Exception in server main event loop running command %s (%s)' % (command, stack))
logger.exception('Exception in server main event loop running command %s (%s)' % (command, stack))
if self.xmlrpc in ready:
self.xmlrpc.handle_requests()
if not seendata and hasattr(self.cooker, "data"):
heartbeat_event = self.cooker.data.getVar('BB_HEARTBEAT_EVENT')
if heartbeat_event:
try:
self.heartbeat_seconds = float(heartbeat_event)
except:
bb.warn('Ignoring invalid BB_HEARTBEAT_EVENT=%s, must be a float specifying seconds.' % heartbeat_event)
self.timeout = self.server_timeout or self.cooker.data.getVar('BB_SERVER_TIMEOUT')
try:
if self.timeout:
self.timeout = float(self.timeout)
except:
bb.warn('Ignoring invalid BB_SERVER_TIMEOUT=%s, must be a float specifying seconds.' % self.timeout)
seendata = True
ready = self.idle_commands(.1, fds)
if self.idle:
self.idle.join()
serverlog("Exiting (socket: %s)" % os.path.exists(self.sockname))
# Remove the socket file so we don't get any more connections to avoid races
# The build directory could have been renamed so if the file isn't the one we created
# we shouldn't delete it.
try:
sockinode = os.stat(self.sockname)[stat.ST_INO]
if sockinode == self.sockinode:
os.unlink(self.sockname)
else:
serverlog("bitbake.sock inode mismatch (%s vs %s), not deleting." % (sockinode, self.sockinode))
except Exception as err:
serverlog("Removing socket file '%s' failed (%s)" % (self.sockname, err))
self.sock.close()
try:
self.cooker.shutdown(True, idle=False)
self.cooker.notifier.stop()
self.cooker.confignotifier.stop()
except:
pass
self.cooker.post_serve()
if len(threading.enumerate()) != 1:
serverlog("More than one thread left?: " + str(threading.enumerate()))
# Flush logs before we release the lock
sys.stdout.flush()
sys.stderr.flush()
# Finally release the lockfile but warn about other processes holding it open
lock = self.bitbake_lock
lockfile = self.bitbake_lock_name
def get_lock_contents(lockfile):
try:
with open(lockfile, "r") as f:
return f.readlines()
except FileNotFoundError:
return None
lock.close()
lock = None
while not lock:
i = 0
lock = None
if not os.path.exists(os.path.basename(lockfile)):
serverlog("Lockfile directory gone, exiting.")
return
while not lock and i < 30:
lock = bb.utils.lockfile(lockfile, shared=False, retry=False, block=False)
if not lock:
newlockcontents = get_lock_contents(lockfile)
if not newlockcontents[0].startswith([os.getpid() + "\n", os.getpid() + " "]):
# A new server was started, the lockfile contents changed, we can exit
serverlog("Lockfile now contains different contents, exiting: " + str(newlockcontents))
return
time.sleep(0.1)
i += 1
if lock:
# We hold the lock so we can remove the file (hide stale pid data)
# via unlockfile.
bb.utils.unlockfile(lock)
serverlog("Exiting as we could obtain the lock")
return
if not lock:
procs = get_lockfile_process_msg(lockfile)
msg = ["Delaying shutdown due to active processes which appear to be holding bitbake.lock"]
if procs:
msg.append(":\n%s" % procs)
serverlog("".join(msg))
def idle_thread(self):
def remove_idle_func(function):
with bb.utils.lock_timeout(self._idlefuncsLock):
del self._idlefuns[function]
self.idle_cond.notify_all()
while not self.quit:
nextsleep = 0.1
fds = []
try:
self.cooker.process_inotify_updates()
except Exception as exc:
serverlog("Exception %s in inofify updates broke the idle_thread, exiting" % traceback.format_exc())
self.quit = True
with bb.utils.lock_timeout(self._idlefuncsLock):
items = list(self._idlefuns.items())
for function, data in items:
try:
retval = function(self, data, False)
if isinstance(retval, idleFinish):
serverlog("Removing idle function %s at idleFinish" % str(function))
remove_idle_func(function)
self.cooker.command.finishAsyncCommand(retval.msg)
nextsleep = None
elif retval is False:
serverlog("Removing idle function %s" % str(function))
remove_idle_func(function)
nextsleep = None
elif retval is True:
nextsleep = None
elif isinstance(retval, float) and nextsleep:
if (retval < nextsleep):
nextsleep = retval
elif nextsleep is None:
continue
else:
fds = fds + retval
except SystemExit:
raise
except Exception as exc:
if not isinstance(exc, bb.BBHandledException):
logger.exception('Running idle function')
remove_idle_func(function)
serverlog("Exception %s broke the idle_thread, exiting" % traceback.format_exc())
self.quit = True
# Create new heartbeat event?
now = time.time()
if bb.event._heartbeat_enabled and now >= self.next_heartbeat:
# We might have missed heartbeats. Just trigger once in
# that case and continue after the usual delay.
self.next_heartbeat += self.heartbeat_seconds
if self.next_heartbeat <= now:
self.next_heartbeat = now + self.heartbeat_seconds
if hasattr(self.cooker, "data"):
heartbeat = bb.event.HeartbeatEvent(now)
try:
bb.event.fire(heartbeat, self.cooker.data)
except Exception as exc:
if not isinstance(exc, bb.BBHandledException):
logger.exception('Running heartbeat function')
serverlog("Exception %s broke in idle_thread, exiting" % traceback.format_exc())
self.quit = True
if nextsleep and bb.event._heartbeat_enabled and now + nextsleep > self.next_heartbeat:
# Shorten timeout so that we we wake up in time for
# the heartbeat.
nextsleep = self.next_heartbeat - now
if nextsleep is not None:
select.select(fds,[],[],nextsleep)[0]
def idle_commands(self, delay, fds=None):
nextsleep = delay
if not fds:
fds = []
if not self.idle:
self.idle = threading.Thread(target=self.idle_thread)
self.idle.start()
elif self.idle and not self.idle.is_alive():
serverlog("Idle thread terminated, main thread exiting too")
bb.error("Idle thread terminated, main thread exiting too")
self.quit = True
if nextsleep is not None:
if self.xmlrpc:
nextsleep = self.xmlrpc.get_timeout(nextsleep)
try:
return select.select(fds,[],[],nextsleep)[0]
except InterruptedError:
# Ignore EINTR
return []
else:
return select.select(fds,[],[],0)[0]
class ServerCommunicator():
def __init__(self, connection, recv):
self.connection = connection
self.recv = recv
def runCommand(self, command):
self.connection.send(command)
if not self.recv.poll(30):
logger.info("No reply from server in 30s")
if not self.recv.poll(30):
raise ProcessTimeout("Timeout while waiting for a reply from the bitbake server (60s)")
ret, exc = self.recv.get()
# Should probably turn all exceptions in exc back into exceptions?
# For now, at least handle BBHandledException
if exc and ("BBHandledException" in exc or "SystemExit" in exc):
raise bb.BBHandledException()
return ret, exc
def updateFeatureSet(self, featureset):
_, error = self.runCommand(["setFeatures", featureset])
if error:
logger.error("Unable to set the cooker to the correct featureset: %s" % error)
raise BaseException(error)
def getEventHandle(self):
handle, error = self.runCommand(["getUIHandlerNum"])
if error:
logger.error("Unable to get UI Handler Number: %s" % error)
raise BaseException(error)
return handle
def terminateServer(self):
self.connection.send(['terminateServer'])
return
class BitBakeProcessServerConnection(object):
def __init__(self, ui_channel, recv, eq, sock):
self.connection = ServerCommunicator(ui_channel, recv)
self.events = eq
# Save sock so it doesn't get gc'd for the life of our connection
self.socket_connection = sock
def terminate(self):
self.events.close()
self.socket_connection.close()
self.connection.connection.close()
self.connection.recv.close()
return
start_log_format = '--- Starting bitbake server pid %s at %s ---'
start_log_datetime_format = '%Y-%m-%d %H:%M:%S.%f'
class BitBakeServer(object):
def __init__(self, lock, sockname, featureset, server_timeout, xmlrpcinterface, profile):
self.server_timeout = server_timeout
self.xmlrpcinterface = xmlrpcinterface
self.featureset = featureset
self.sockname = sockname
self.bitbake_lock = lock
self.profile = profile
self.readypipe, self.readypipein = os.pipe()
# Place the log in the builddirectory alongside the lock file
logfile = os.path.join(os.path.dirname(self.bitbake_lock.name), "bitbake-cookerdaemon.log")
self.logfile = logfile
startdatetime = datetime.datetime.now()
bb.daemonize.createDaemon(self._startServer, logfile)
self.bitbake_lock.close()
os.close(self.readypipein)
ready = ConnectionReader(self.readypipe)
r = ready.poll(5)
if not r:
bb.note("Bitbake server didn't start within 5 seconds, waiting for 90")
r = ready.poll(90)
if r:
try:
r = ready.get()
except EOFError:
# Trap the child exiting/closing the pipe and error out
r = None
if not r or r[0] != "r":
ready.close()
bb.error("Unable to start bitbake server (%s)" % str(r))
if os.path.exists(logfile):
logstart_re = re.compile(start_log_format % ('([0-9]+)', '([0-9-]+ [0-9:.]+)'))
started = False
lines = []
lastlines = []
with open(logfile, "r") as f:
for line in f:
if started:
lines.append(line)
else:
lastlines.append(line)
res = logstart_re.search(line.rstrip())
if res:
ldatetime = datetime.datetime.strptime(res.group(2), start_log_datetime_format)
if ldatetime >= startdatetime:
started = True
lines.append(line)
if len(lastlines) > 60:
lastlines = lastlines[-60:]
if lines:
if len(lines) > 60:
bb.error("Last 60 lines of server log for this session (%s):\n%s" % (logfile, "".join(lines[-60:])))
else:
bb.error("Server log for this session (%s):\n%s" % (logfile, "".join(lines)))
elif lastlines:
bb.error("Server didn't start, last 60 loglines (%s):\n%s" % (logfile, "".join(lastlines)))
else:
bb.error("%s doesn't exist" % logfile)
raise SystemExit(1)
ready.close()
def _startServer(self):
os.close(self.readypipe)
os.set_inheritable(self.bitbake_lock.fileno(), True)
os.set_inheritable(self.readypipein, True)
serverscript = os.path.realpath(os.path.dirname(__file__) + "/../../../bin/bitbake-server")
os.execl(sys.executable, "bitbake-server", serverscript, "decafbad", str(self.bitbake_lock.fileno()), str(self.readypipein), self.logfile, self.bitbake_lock.name, self.sockname, str(self.server_timeout or 0), str(int(self.profile)), str(self.xmlrpcinterface[0]), str(self.xmlrpcinterface[1]))
def execServer(lockfd, readypipeinfd, lockname, sockname, server_timeout, xmlrpcinterface, profile):
import bb.cookerdata
import bb.cooker
serverlog(start_log_format % (os.getpid(), datetime.datetime.now().strftime(start_log_datetime_format)))
try:
bitbake_lock = os.fdopen(lockfd, "w")
# Create server control socket
if os.path.exists(sockname):
serverlog("WARNING: removing existing socket file '%s'" % sockname)
os.unlink(sockname)
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
# AF_UNIX has path length issues so chdir here to workaround
cwd = os.getcwd()
try:
os.chdir(os.path.dirname(sockname))
sock.bind(os.path.basename(sockname))
finally:
os.chdir(cwd)
sock.listen(1)
server = ProcessServer(bitbake_lock, lockname, sock, sockname, server_timeout, xmlrpcinterface)
writer = ConnectionWriter(readypipeinfd)
try:
featureset = []
cooker = bb.cooker.BBCooker(featureset, server)
cooker.configuration.profile = profile
except bb.BBHandledException:
return None
writer.send("r")
writer.close()
server.cooker = cooker
serverlog("Started bitbake server pid %d" % os.getpid())
server.run()
finally:
# Flush any messages/errors to the logfile before exit
sys.stdout.flush()
sys.stderr.flush()
def connectProcessServer(sockname, featureset):
# Connect to socket
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
# AF_UNIX has path length issues so chdir here to workaround
cwd = os.getcwd()
readfd = writefd = readfd1 = writefd1 = readfd2 = writefd2 = None
eq = command_chan_recv = command_chan = None
sock.settimeout(10)
try:
try:
os.chdir(os.path.dirname(sockname))
finished = False
while not finished:
try:
sock.connect(os.path.basename(sockname))
finished = True
except IOError as e:
if e.errno == errno.EWOULDBLOCK:
pass
raise
finally:
os.chdir(cwd)
# Send an fd for the remote to write events to
readfd, writefd = os.pipe()
eq = BBUIEventQueue(readfd)
# Send an fd for the remote to recieve commands from
readfd1, writefd1 = os.pipe()
command_chan = ConnectionWriter(writefd1)
# Send an fd for the remote to write commands results to
readfd2, writefd2 = os.pipe()
command_chan_recv = ConnectionReader(readfd2)
sendfds(sock, [writefd, readfd1, writefd2])
server_connection = BitBakeProcessServerConnection(command_chan, command_chan_recv, eq, sock)
# Close the ends of the pipes we won't use
for i in [writefd, readfd1, writefd2]:
os.close(i)
server_connection.connection.updateFeatureSet(featureset)
except (Exception, SystemExit) as e:
if command_chan_recv:
command_chan_recv.close()
if command_chan:
command_chan.close()
for i in [writefd, readfd1, writefd2]:
try:
if i:
os.close(i)
except OSError:
pass
sock.close()
raise
return server_connection
def sendfds(sock, fds):
'''Send an array of fds over an AF_UNIX socket.'''
fds = array.array('i', fds)
msg = bytes([len(fds) % 256])
sock.sendmsg([msg], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, fds)])
def recvfds(sock, size):
'''Receive an array of fds over an AF_UNIX socket.'''
a = array.array('i')
bytes_size = a.itemsize * size
msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size))
if not msg and not ancdata:
raise EOFError
try:
if len(ancdata) != 1:
raise RuntimeError('received %d items of ancdata' %
len(ancdata))
cmsg_level, cmsg_type, cmsg_data = ancdata[0]
if (cmsg_level == socket.SOL_SOCKET and
cmsg_type == socket.SCM_RIGHTS):
if len(cmsg_data) % a.itemsize != 0:
raise ValueError
a.frombytes(cmsg_data)
assert len(a) % 256 == msg[0]
return list(a)
except (ValueError, IndexError):
pass
raise RuntimeError('Invalid data received')
class BBUIEventQueue:
def __init__(self, readfd):
self.eventQueue = []
self.eventQueueLock = threading.Lock()
self.eventQueueNotify = threading.Event()
self.reader = ConnectionReader(readfd)
self.t = threading.Thread()
self.t.run = self.startCallbackHandler
self.t.start()
def getEvent(self):
with bb.utils.lock_timeout(self.eventQueueLock):
if len(self.eventQueue) == 0:
return None
item = self.eventQueue.pop(0)
if len(self.eventQueue) == 0:
self.eventQueueNotify.clear()
return item
def waitEvent(self, delay):
self.eventQueueNotify.wait(delay)
return self.getEvent()
def queue_event(self, event):
with bb.utils.lock_timeout(self.eventQueueLock):
self.eventQueue.append(event)
self.eventQueueNotify.set()
def send_event(self, event):
self.queue_event(pickle.loads(event))
def startCallbackHandler(self):
bb.utils.set_process_name("UIEventQueue")
while True:
try:
ready = self.reader.wait(0.25)
if ready:
event = self.reader.get()
self.queue_event(event)
except (EOFError, OSError, TypeError):
# Easiest way to exit is to close the file descriptor to cause an exit
break
def close(self):
self.reader.close()
self.t.join()
class ConnectionReader(object):
def __init__(self, fd):
self.reader = multiprocessing.connection.Connection(fd, writable=False)
self.rlock = multiprocessing.Lock()
def wait(self, timeout=None):
return multiprocessing.connection.wait([self.reader], timeout)
def poll(self, timeout=None):
return self.reader.poll(timeout)
def get(self):
with bb.utils.lock_timeout(self.rlock):
res = self.reader.recv_bytes()
return multiprocessing.reduction.ForkingPickler.loads(res)
def fileno(self):
return self.reader.fileno()
def close(self):
return self.reader.close()
class ConnectionWriter(object):
def __init__(self, fd):
self.writer = multiprocessing.connection.Connection(fd, readable=False)
self.wlock = multiprocessing.Lock()
# Why bb.event needs this I have no idea
self.event = self
def _send(self, obj):
gc.disable()
with bb.utils.lock_timeout(self.wlock):
self.writer.send_bytes(obj)
gc.enable()
def send(self, obj):
obj = multiprocessing.reduction.ForkingPickler.dumps(obj)
# See notes/code in CookerParser
# We must not terminate holding this lock else processes will hang.
# For SIGTERM, raising afterwards avoids this.
# For SIGINT, we don't want to have written partial data to the pipe.
# pthread_sigmask block/unblock would be nice but doesn't work, https://bugs.python.org/issue47139
process = multiprocessing.current_process()
if process and hasattr(process, "queue_signals"):
with bb.utils.lock_timeout(process.signal_threadlock):
process.queue_signals = True
self._send(obj)
process.queue_signals = False
try:
for sig in process.signal_received.pop():
process.handle_sig(sig, None)
except IndexError:
pass
else:
self._send(obj)
def fileno(self):
return self.writer.fileno()
def close(self):
return self.writer.close()