poky: refresh thud: cc73390a75..3541f019a5
Update poky to thud HEAD:
Alejandro Enedino Hernandez Samaniego (1):
bitbake: cooker: fix indirect multiconfig dependencies
Richard Purdie (8):
bitbake: server/process: Increase server startup time delay
bitbake: cooker: Add some timing debug messages to the server startup
bitbake: server/process: Handle short reads
bitbake: lib/bb/server: Avoid UnboundLocalError traceback
bitbake: server/process: Ensure socket has a timeout set
bitbake: process.py: Set socket timeout to 10 seconds
bitbake: process: Handle EWOULDBLOCK in socket connect
bitbake: process: Rewrite multiple connection handling
Change-Id: I175bb3a51cc19f3aef5ffb6e09b04cf0bf882835
Signed-off-by: Brad Bishop <bradleyb@fuzziesquirrel.com>
diff --git a/poky/bitbake/lib/bb/cooker.py b/poky/bitbake/lib/bb/cooker.py
index 16681ba..db52964 100644
--- a/poky/bitbake/lib/bb/cooker.py
+++ b/poky/bitbake/lib/bb/cooker.py
@@ -175,18 +175,31 @@
self.configuration = configuration
+ bb.debug(1, "BBCooker starting %s" % time.time())
+ sys.stdout.flush()
+
self.configwatcher = pyinotify.WatchManager()
+ bb.debug(1, "BBCooker pyinotify1 %s" % time.time())
+ sys.stdout.flush()
+
self.configwatcher.bbseen = []
self.configwatcher.bbwatchedfiles = []
self.confignotifier = pyinotify.Notifier(self.configwatcher, self.config_notifications)
+ bb.debug(1, "BBCooker pyinotify2 %s" % time.time())
+ sys.stdout.flush()
self.watchmask = pyinotify.IN_CLOSE_WRITE | pyinotify.IN_CREATE | pyinotify.IN_DELETE | \
pyinotify.IN_DELETE_SELF | pyinotify.IN_MODIFY | pyinotify.IN_MOVE_SELF | \
pyinotify.IN_MOVED_FROM | pyinotify.IN_MOVED_TO
self.watcher = pyinotify.WatchManager()
+ bb.debug(1, "BBCooker pyinotify3 %s" % time.time())
+ sys.stdout.flush()
self.watcher.bbseen = []
self.watcher.bbwatchedfiles = []
self.notifier = pyinotify.Notifier(self.watcher, self.notifications)
+ bb.debug(1, "BBCooker pyinotify complete %s" % time.time())
+ sys.stdout.flush()
+
# If being called by something like tinfoil, we need to clean cached data
# which may now be invalid
bb.parse.clear_cache()
@@ -196,6 +209,9 @@
self.initConfigurationData()
+ bb.debug(1, "BBCooker parsed base configuration %s" % time.time())
+ sys.stdout.flush()
+
# we log all events to a file if so directed
if self.configuration.writeeventlog:
# register the log file writer as UI Handler
@@ -233,6 +249,9 @@
# Let SIGHUP exit as SIGTERM
signal.signal(signal.SIGHUP, self.sigterm_exception)
+ bb.debug(1, "BBCooker startup complete %s" % time.time())
+ sys.stdout.flush()
+
def process_inotify_updates(self):
for n in [self.confignotifier, self.notifier]:
if n.check_events(timeout=0):
@@ -620,27 +639,38 @@
runlist.append([mc, k, ktask, fn])
bb.event.fire(bb.event.TreeDataPreparationProgress(current, len(fulltargetlist)), self.data)
- mcdeps = taskdata[mc].get_mcdepends()
+
# No need to do check providers if there are no mcdeps or not an mc build
- if mcdeps and mc:
- # Make sure we can provide the multiconfig dependency
- seen = set()
- new = True
- while new:
- new = False
- for mc in self.multiconfigs:
- for k in mcdeps:
- if k in seen:
- continue
- l = k.split(':')
- depmc = l[2]
- if depmc not in self.multiconfigs:
- bb.fatal("Multiconfig dependency %s depends on nonexistent mc configuration %s" % (k,depmc))
- else:
- logger.debug(1, "Adding providers for multiconfig dependency %s" % l[3])
- taskdata[depmc].add_provider(localdata[depmc], self.recipecaches[depmc], l[3])
- seen.add(k)
- new = True
+ if mc:
+ # Add unresolved first, so we can get multiconfig indirect dependencies on time
+ for mcavailable in self.multiconfigs:
+ # The first element is empty
+ if mcavailable:
+ taskdata[mcavailable].add_unresolved(localdata[mcavailable], self.recipecaches[mcavailable])
+
+
+ mcdeps = taskdata[mc].get_mcdepends()
+
+ if mcdeps:
+ # Make sure we can provide the multiconfig dependency
+ seen = set()
+ new = True
+ while new:
+ new = False
+ for mc in self.multiconfigs:
+ for k in mcdeps:
+ if k in seen:
+ continue
+ l = k.split(':')
+ depmc = l[2]
+ if depmc not in self.multiconfigs:
+ bb.fatal("Multiconfig dependency %s depends on nonexistent mc configuration %s" % (k,depmc))
+ else:
+ logger.debug(1, "Adding providers for multiconfig dependency %s" % l[3])
+ taskdata[depmc].add_provider(localdata[depmc], self.recipecaches[depmc], l[3])
+ seen.add(k)
+ new = True
+
for mc in self.multiconfigs:
taskdata[mc].add_unresolved(localdata[mc], self.recipecaches[mc])
diff --git a/poky/bitbake/lib/bb/server/process.py b/poky/bitbake/lib/bb/server/process.py
index 070da4f..28b8eb9 100644
--- a/poky/bitbake/lib/bb/server/process.py
+++ b/poky/bitbake/lib/bb/server/process.py
@@ -130,6 +130,7 @@
bb.utils.set_process_name("Cooker")
ready = []
+ newconnections = []
self.controllersock = False
fds = [self.sock]
@@ -138,37 +139,48 @@
print("Entering server connection loop")
def disconnect_client(self, fds):
- if not self.haveui:
- return
print("Disconnecting Client")
- fds.remove(self.controllersock)
- fds.remove(self.command_channel)
- bb.event.unregister_UIHhandler(self.event_handle, True)
- self.command_channel_reply.writer.close()
- self.event_writer.writer.close()
- del self.event_writer
- self.controllersock.close()
- self.controllersock = False
- self.haveui = False
- self.lastui = time.time()
- self.cooker.clientComplete()
- if self.timeout is None:
+ if self.controllersock:
+ fds.remove(self.controllersock)
+ self.controllersock.close()
+ self.controllersock = False
+ if self.haveui:
+ fds.remove(self.command_channel)
+ bb.event.unregister_UIHhandler(self.event_handle, True)
+ self.command_channel_reply.writer.close()
+ self.event_writer.writer.close()
+ self.command_channel.close()
+ self.command_channel = False
+ del self.event_writer
+ self.lastui = time.time()
+ self.cooker.clientComplete()
+ self.haveui = False
+ ready = select.select(fds,[],[],0)[0]
+ if newconnections:
+ print("Starting new client")
+ conn = newconnections.pop(-1)
+ fds.append(conn)
+ self.controllersock = conn
+ elif self.timeout is None and not ready:
print("No timeout, exiting.")
self.quit = True
while not self.quit:
if self.sock in ready:
- self.controllersock, address = self.sock.accept()
- if self.haveui:
- print("Dropping connection attempt as we have a UI %s" % (str(ready)))
- self.controllersock.close()
- else:
- print("Accepting %s" % (str(ready)))
- fds.append(self.controllersock)
+ while select.select([self.sock],[],[],0)[0]:
+ controllersock, address = self.sock.accept()
+ if self.controllersock:
+ print("Queuing %s (%s)" % (str(ready), str(newconnections)))
+ newconnections.append(controllersock)
+ else:
+ print("Accepting %s (%s)" % (str(ready), str(newconnections)))
+ self.controllersock = controllersock
+ fds.append(controllersock)
if self.controllersock in ready:
try:
- print("Connecting Client")
+ print("Processing Client")
ui_fds = recvfds(self.controllersock, 3)
+ print("Connecting Client")
# Where to write events to
writer = ConnectionWriter(ui_fds[0])
@@ -400,16 +412,19 @@
os.close(self.readypipein)
ready = ConnectionReader(self.readypipe)
- r = ready.poll(30)
+ r = ready.poll(5)
+ if not r:
+ bb.note("Bitbake server didn't start within 5 seconds, waiting for 90")
+ r = ready.poll(90)
if r:
try:
r = ready.get()
except EOFError:
# Trap the child exitting/closing the pipe and error out
r = None
- if not r or r != "ready":
+ if not r or r[0] != "r":
ready.close()
- bb.error("Unable to start bitbake server")
+ bb.error("Unable to start bitbake server (%s)" % str(r))
if os.path.exists(logfile):
logstart_re = re.compile(self.start_log_format % ('([0-9]+)', '([0-9-]+ [0-9:.]+)'))
started = False
@@ -452,7 +467,7 @@
os.close(self.readypipe)
writer = ConnectionWriter(self.readypipein)
self.cooker = bb.cooker.BBCooker(self.configuration, self.featureset)
- writer.send("ready")
+ writer.send("r")
writer.close()
server.cooker = self.cooker
server.server_timeout = self.configuration.server_timeout
@@ -468,16 +483,25 @@
# AF_UNIX has path length issues so chdir here to workaround
cwd = os.getcwd()
+ readfd = writefd = readfd1 = writefd1 = readfd2 = writefd2 = None
+ eq = command_chan_recv = command_chan = None
+
+ sock.settimeout(10)
+
try:
try:
os.chdir(os.path.dirname(sockname))
- sock.connect(os.path.basename(sockname))
+ finished = False
+ while not finished:
+ try:
+ sock.connect(os.path.basename(sockname))
+ finished = True
+ except IOError as e:
+ if e.errno == errno.EWOULDBLOCK:
+ pass
finally:
os.chdir(cwd)
- readfd = writefd = readfd1 = writefd1 = readfd2 = writefd2 = None
- eq = command_chan_recv = command_chan = None
-
# Send an fd for the remote to write events to
readfd, writefd = os.pipe()
eq = BBUIEventQueue(readfd)