Squashed 'yocto-poky/' content from commit ea562de
git-subtree-dir: yocto-poky
git-subtree-split: ea562de57590c966cd5a75fda8defecd397e6436
diff --git a/bitbake/lib/bb/server/__init__.py b/bitbake/lib/bb/server/__init__.py
new file mode 100644
index 0000000..da5e480
--- /dev/null
+++ b/bitbake/lib/bb/server/__init__.py
@@ -0,0 +1,96 @@
+#
+# BitBake Base Server Code
+#
+# Copyright (C) 2006 - 2007 Michael 'Mickey' Lauer
+# Copyright (C) 2006 - 2008 Richard Purdie
+# Copyright (C) 2013 Alexandru Damian
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+""" Base code for Bitbake server process
+
+Have a common base for that all Bitbake server classes ensures a consistent
+approach to the interface, and minimize risks associated with code duplication.
+
+"""
+
+""" BaseImplServer() the base class for all XXServer() implementations.
+
+ These classes contain the actual code that runs the server side, i.e.
+ listens for the commands and executes them. Although these implementations
+ contain all the data of the original bitbake command, i.e the cooker instance,
+ they may well run on a different process or even machine.
+
+"""
+
+class BaseImplServer():
+ def __init__(self):
+ self._idlefuns = {}
+
+ def addcooker(self, cooker):
+ self.cooker = cooker
+
+ def register_idle_function(self, function, data):
+ """Register a function to be called while the server is idle"""
+ assert hasattr(function, '__call__')
+ self._idlefuns[function] = data
+
+
+
+""" BitBakeBaseServerConnection class is the common ancestor to all
+ BitBakeServerConnection classes.
+
+ These classes control the remote server. The only command currently
+ implemented is the terminate() command.
+
+"""
+
+class BitBakeBaseServerConnection():
+ def __init__(self, serverImpl):
+ pass
+
+ def terminate(self):
+ pass
+
+
+""" BitBakeBaseServer class is the common ancestor to all Bitbake servers
+
+ Derive this class in order to implement a BitBakeServer which is the
+ controlling stub for the actual server implementation
+
+"""
+class BitBakeBaseServer(object):
+ def initServer(self):
+ self.serverImpl = None # we ensure a runtime crash if not overloaded
+ self.connection = None
+ return
+
+ def addcooker(self, cooker):
+ self.cooker = cooker
+ self.serverImpl.addcooker(cooker)
+
+ def getServerIdleCB(self):
+ return self.serverImpl.register_idle_function
+
+ def saveConnectionDetails(self):
+ return
+
+ def detach(self):
+ return
+
+ def establishConnection(self, featureset):
+ raise "Must redefine the %s.establishConnection()" % self.__class__.__name__
+
+ def endSession(self):
+ self.connection.terminate()
diff --git a/bitbake/lib/bb/server/process.py b/bitbake/lib/bb/server/process.py
new file mode 100644
index 0000000..5fca350
--- /dev/null
+++ b/bitbake/lib/bb/server/process.py
@@ -0,0 +1,264 @@
+#
+# BitBake Process based server.
+#
+# Copyright (C) 2010 Bob Foerster <robert@erafx.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+"""
+ This module implements a multiprocessing.Process based server for bitbake.
+"""
+
+import bb
+import bb.event
+import itertools
+import logging
+import multiprocessing
+import os
+import signal
+import sys
+import time
+import select
+from Queue import Empty
+from multiprocessing import Event, Process, util, Queue, Pipe, queues, Manager
+
+from . import BitBakeBaseServer, BitBakeBaseServerConnection, BaseImplServer
+
+logger = logging.getLogger('BitBake')
+
+class ServerCommunicator():
+ def __init__(self, connection, event_handle, server):
+ self.connection = connection
+ self.event_handle = event_handle
+ self.server = server
+
+ def runCommand(self, command):
+ # @todo try/except
+ self.connection.send(command)
+
+ if not self.server.is_alive():
+ raise SystemExit
+
+ while True:
+ # don't let the user ctrl-c while we're waiting for a response
+ try:
+ if self.connection.poll(20):
+ return self.connection.recv()
+ else:
+ bb.fatal("Timeout while attempting to communicate with bitbake server")
+ except KeyboardInterrupt:
+ pass
+
+ def getEventHandle(self):
+ return self.event_handle.value
+
+class EventAdapter():
+ """
+ Adapter to wrap our event queue since the caller (bb.event) expects to
+ call a send() method, but our actual queue only has put()
+ """
+ def __init__(self, queue):
+ self.queue = queue
+
+ def send(self, event):
+ try:
+ self.queue.put(event)
+ except Exception as err:
+ print("EventAdapter puked: %s" % str(err))
+
+
+class ProcessServer(Process, BaseImplServer):
+ profile_filename = "profile.log"
+ profile_processed_filename = "profile.log.processed"
+
+ def __init__(self, command_channel, event_queue, featurelist):
+ BaseImplServer.__init__(self)
+ Process.__init__(self)
+ self.command_channel = command_channel
+ self.event_queue = event_queue
+ self.event = EventAdapter(event_queue)
+ self.featurelist = featurelist
+ self.quit = False
+
+ self.quitin, self.quitout = Pipe()
+ self.event_handle = multiprocessing.Value("i")
+
+ def run(self):
+ for event in bb.event.ui_queue:
+ self.event_queue.put(event)
+ self.event_handle.value = bb.event.register_UIHhandler(self, True)
+
+ bb.cooker.server_main(self.cooker, self.main)
+
+ def main(self):
+ # Ignore SIGINT within the server, as all SIGINT handling is done by
+ # the UI and communicated to us
+ self.quitin.close()
+ signal.signal(signal.SIGINT, signal.SIG_IGN)
+ while not self.quit:
+ try:
+ if self.command_channel.poll():
+ command = self.command_channel.recv()
+ self.runCommand(command)
+ if self.quitout.poll():
+ self.quitout.recv()
+ self.quit = True
+ try:
+ self.runCommand(["stateForceShutdown"])
+ except:
+ pass
+
+ self.idle_commands(.1, [self.command_channel, self.quitout])
+ except Exception:
+ logger.exception('Running command %s', command)
+
+ self.event_queue.close()
+ bb.event.unregister_UIHhandler(self.event_handle.value)
+ self.command_channel.close()
+ self.cooker.shutdown(True)
+ self.quitout.close()
+
+ def idle_commands(self, delay, fds=None):
+ nextsleep = delay
+ if not fds:
+ fds = []
+
+ for function, data in self._idlefuns.items():
+ try:
+ retval = function(self, data, False)
+ if retval is False:
+ del self._idlefuns[function]
+ nextsleep = None
+ elif retval is True:
+ nextsleep = None
+ elif isinstance(retval, float):
+ if (retval < nextsleep):
+ nextsleep = retval
+ elif nextsleep is None:
+ continue
+ else:
+ fds = fds + retval
+ except SystemExit:
+ raise
+ except Exception as exc:
+ if not isinstance(exc, bb.BBHandledException):
+ logger.exception('Running idle function')
+ del self._idlefuns[function]
+ self.quit = True
+
+ if nextsleep is not None:
+ select.select(fds,[],[],nextsleep)
+
+ def runCommand(self, command):
+ """
+ Run a cooker command on the server
+ """
+ self.command_channel.send(self.cooker.command.runCommand(command))
+
+ def stop(self):
+ self.quitin.send("quit")
+ self.quitin.close()
+
+class BitBakeProcessServerConnection(BitBakeBaseServerConnection):
+ def __init__(self, serverImpl, ui_channel, event_queue):
+ self.procserver = serverImpl
+ self.ui_channel = ui_channel
+ self.event_queue = event_queue
+ self.connection = ServerCommunicator(self.ui_channel, self.procserver.event_handle, self.procserver)
+ self.events = self.event_queue
+ self.terminated = False
+
+ def sigterm_terminate(self):
+ bb.error("UI received SIGTERM")
+ self.terminate()
+
+ def terminate(self):
+ if self.terminated:
+ return
+ self.terminated = True
+ def flushevents():
+ while True:
+ try:
+ event = self.event_queue.get(block=False)
+ except (Empty, IOError):
+ break
+ if isinstance(event, logging.LogRecord):
+ logger.handle(event)
+
+ signal.signal(signal.SIGINT, signal.SIG_IGN)
+ self.procserver.stop()
+
+ while self.procserver.is_alive():
+ flushevents()
+ self.procserver.join(0.1)
+
+ self.ui_channel.close()
+ self.event_queue.close()
+ self.event_queue.setexit()
+
+# Wrap Queue to provide API which isn't server implementation specific
+class ProcessEventQueue(multiprocessing.queues.Queue):
+ def __init__(self, maxsize):
+ multiprocessing.queues.Queue.__init__(self, maxsize)
+ self.exit = False
+
+ def setexit(self):
+ self.exit = True
+
+ def waitEvent(self, timeout):
+ if self.exit:
+ sys.exit(1)
+ try:
+ if not self.server.is_alive():
+ self.setexit()
+ return None
+ return self.get(True, timeout)
+ except Empty:
+ return None
+
+ def getEvent(self):
+ try:
+ if not self.server.is_alive():
+ self.setexit()
+ return None
+ return self.get(False)
+ except Empty:
+ return None
+
+
+class BitBakeServer(BitBakeBaseServer):
+ def initServer(self):
+ # establish communication channels. We use bidirectional pipes for
+ # ui <--> server command/response pairs
+ # and a queue for server -> ui event notifications
+ #
+ self.ui_channel, self.server_channel = Pipe()
+ self.event_queue = ProcessEventQueue(0)
+ self.serverImpl = ProcessServer(self.server_channel, self.event_queue, None)
+ self.event_queue.server = self.serverImpl
+
+ def detach(self):
+ self.serverImpl.start()
+ return
+
+ def establishConnection(self, featureset):
+
+ self.connection = BitBakeProcessServerConnection(self.serverImpl, self.ui_channel, self.event_queue)
+
+ _, error = self.connection.connection.runCommand(["setFeatures", featureset])
+ if error:
+ logger.error("Unable to set the cooker to the correct featureset: %s" % error)
+ raise BaseException(error)
+ signal.signal(signal.SIGTERM, lambda i, s: self.connection.sigterm_terminate())
+ return self.connection
diff --git a/bitbake/lib/bb/server/xmlrpc.py b/bitbake/lib/bb/server/xmlrpc.py
new file mode 100644
index 0000000..b7647c1
--- /dev/null
+++ b/bitbake/lib/bb/server/xmlrpc.py
@@ -0,0 +1,392 @@
+#
+# BitBake XMLRPC Server
+#
+# Copyright (C) 2006 - 2007 Michael 'Mickey' Lauer
+# Copyright (C) 2006 - 2008 Richard Purdie
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+"""
+ This module implements an xmlrpc server for BitBake.
+
+ Use this by deriving a class from BitBakeXMLRPCServer and then adding
+ methods which you want to "export" via XMLRPC. If the methods have the
+ prefix xmlrpc_, then registering those function will happen automatically,
+ if not, you need to call register_function.
+
+ Use register_idle_function() to add a function which the xmlrpc server
+ calls from within server_forever when no requests are pending. Make sure
+ that those functions are non-blocking or else you will introduce latency
+ in the server's main loop.
+"""
+
+import bb
+import xmlrpclib, sys
+from bb import daemonize
+from bb.ui import uievent
+import hashlib, time
+import socket
+import os, signal
+import threading
+try:
+ import cPickle as pickle
+except ImportError:
+ import pickle
+
+DEBUG = False
+
+from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
+import inspect, select, httplib
+
+from . import BitBakeBaseServer, BitBakeBaseServerConnection, BaseImplServer
+
+class BBTransport(xmlrpclib.Transport):
+ def __init__(self, timeout):
+ self.timeout = timeout
+ self.connection_token = None
+ xmlrpclib.Transport.__init__(self)
+
+ # Modified from default to pass timeout to HTTPConnection
+ def make_connection(self, host):
+ #return an existing connection if possible. This allows
+ #HTTP/1.1 keep-alive.
+ if self._connection and host == self._connection[0]:
+ return self._connection[1]
+
+ # create a HTTP connection object from a host descriptor
+ chost, self._extra_headers, x509 = self.get_host_info(host)
+ #store the host argument along with the connection object
+ self._connection = host, httplib.HTTPConnection(chost, timeout=self.timeout)
+ return self._connection[1]
+
+ def set_connection_token(self, token):
+ self.connection_token = token
+
+ def send_content(self, h, body):
+ if self.connection_token:
+ h.putheader("Bitbake-token", self.connection_token)
+ xmlrpclib.Transport.send_content(self, h, body)
+
+def _create_server(host, port, timeout = 60):
+ t = BBTransport(timeout)
+ s = xmlrpclib.ServerProxy("http://%s:%d/" % (host, port), transport=t, allow_none=True)
+ return s, t
+
+class BitBakeServerCommands():
+
+ def __init__(self, server):
+ self.server = server
+ self.has_client = False
+
+ def registerEventHandler(self, host, port):
+ """
+ Register a remote UI Event Handler
+ """
+ s, t = _create_server(host, port)
+
+ # we don't allow connections if the cooker is running
+ if (self.cooker.state in [bb.cooker.state.parsing, bb.cooker.state.running]):
+ return None
+
+ self.event_handle = bb.event.register_UIHhandler(s, True)
+ return self.event_handle
+
+ def unregisterEventHandler(self, handlerNum):
+ """
+ Unregister a remote UI Event Handler
+ """
+ return bb.event.unregister_UIHhandler(handlerNum)
+
+ def runCommand(self, command):
+ """
+ Run a cooker command on the server
+ """
+ return self.cooker.command.runCommand(command, self.server.readonly)
+
+ def getEventHandle(self):
+ return self.event_handle
+
+ def terminateServer(self):
+ """
+ Trigger the server to quit
+ """
+ self.server.quit = True
+ print("Server (cooker) exiting")
+ return
+
+ def addClient(self):
+ if self.has_client:
+ return None
+ token = hashlib.md5(str(time.time())).hexdigest()
+ self.server.set_connection_token(token)
+ self.has_client = True
+ return token
+
+ def removeClient(self):
+ if self.has_client:
+ self.server.set_connection_token(None)
+ self.has_client = False
+ if self.server.single_use:
+ self.server.quit = True
+
+# This request handler checks if the request has a "Bitbake-token" header
+# field (this comes from the client side) and compares it with its internal
+# "Bitbake-token" field (this comes from the server). If the two are not
+# equal, it is assumed that a client is trying to connect to the server
+# while another client is connected to the server. In this case, a 503 error
+# ("service unavailable") is returned to the client.
+class BitBakeXMLRPCRequestHandler(SimpleXMLRPCRequestHandler):
+ def __init__(self, request, client_address, server):
+ self.server = server
+ SimpleXMLRPCRequestHandler.__init__(self, request, client_address, server)
+
+ def do_POST(self):
+ try:
+ remote_token = self.headers["Bitbake-token"]
+ except:
+ remote_token = None
+ if remote_token != self.server.connection_token and remote_token != "observer":
+ self.report_503()
+ else:
+ if remote_token == "observer":
+ self.server.readonly = True
+ else:
+ self.server.readonly = False
+ SimpleXMLRPCRequestHandler.do_POST(self)
+
+ def report_503(self):
+ self.send_response(503)
+ response = 'No more client allowed'
+ self.send_header("Content-type", "text/plain")
+ self.send_header("Content-length", str(len(response)))
+ self.end_headers()
+ self.wfile.write(response)
+
+
+class XMLRPCProxyServer(BaseImplServer):
+ """ not a real working server, but a stub for a proxy server connection
+
+ """
+ def __init__(self, host, port):
+ self.host = host
+ self.port = port
+
+class XMLRPCServer(SimpleXMLRPCServer, BaseImplServer):
+ # remove this when you're done with debugging
+ # allow_reuse_address = True
+
+ def __init__(self, interface):
+ """
+ Constructor
+ """
+ BaseImplServer.__init__(self)
+ if (interface[1] == 0): # anonymous port, not getting reused
+ self.single_use = True
+ # Use auto port configuration
+ if (interface[1] == -1):
+ interface = (interface[0], 0)
+ SimpleXMLRPCServer.__init__(self, interface,
+ requestHandler=BitBakeXMLRPCRequestHandler,
+ logRequests=False, allow_none=True)
+ self.host, self.port = self.socket.getsockname()
+ self.connection_token = None
+ #self.register_introspection_functions()
+ self.commands = BitBakeServerCommands(self)
+ self.autoregister_all_functions(self.commands, "")
+ self.interface = interface
+ self.single_use = False
+
+ def addcooker(self, cooker):
+ BaseImplServer.addcooker(self, cooker)
+ self.commands.cooker = cooker
+
+ def autoregister_all_functions(self, context, prefix):
+ """
+ Convenience method for registering all functions in the scope
+ of this class that start with a common prefix
+ """
+ methodlist = inspect.getmembers(context, inspect.ismethod)
+ for name, method in methodlist:
+ if name.startswith(prefix):
+ self.register_function(method, name[len(prefix):])
+
+
+ def serve_forever(self):
+ # Start the actual XMLRPC server
+ bb.cooker.server_main(self.cooker, self._serve_forever)
+
+ def _serve_forever(self):
+ """
+ Serve Requests. Overloaded to honor a quit command
+ """
+ self.quit = False
+ while not self.quit:
+ fds = [self]
+ nextsleep = 0.1
+ for function, data in self._idlefuns.items():
+ retval = None
+ try:
+ retval = function(self, data, False)
+ if retval is False:
+ del self._idlefuns[function]
+ elif retval is True:
+ nextsleep = 0
+ elif isinstance(retval, float):
+ if (retval < nextsleep):
+ nextsleep = retval
+ else:
+ fds = fds + retval
+ except SystemExit:
+ raise
+ except:
+ import traceback
+ traceback.print_exc()
+ if retval == None:
+ # the function execute failed; delete it
+ del self._idlefuns[function]
+ pass
+
+ socktimeout = self.socket.gettimeout() or nextsleep
+ socktimeout = min(socktimeout, nextsleep)
+ # Mirror what BaseServer handle_request would do
+ try:
+ fd_sets = select.select(fds, [], [], socktimeout)
+ if fd_sets[0] and self in fd_sets[0]:
+ self._handle_request_noblock()
+ except IOError:
+ # we ignore interrupted calls
+ pass
+
+ # Tell idle functions we're exiting
+ for function, data in self._idlefuns.items():
+ try:
+ retval = function(self, data, True)
+ except:
+ pass
+ self.server_close()
+ return
+
+ def set_connection_token(self, token):
+ self.connection_token = token
+
+class BitBakeXMLRPCServerConnection(BitBakeBaseServerConnection):
+ def __init__(self, serverImpl, clientinfo=("localhost", 0), observer_only = False, featureset = None):
+ self.connection, self.transport = _create_server(serverImpl.host, serverImpl.port)
+ self.clientinfo = clientinfo
+ self.serverImpl = serverImpl
+ self.observer_only = observer_only
+ if featureset:
+ self.featureset = featureset
+ else:
+ self.featureset = []
+
+ def connect(self, token = None):
+ if token is None:
+ if self.observer_only:
+ token = "observer"
+ else:
+ token = self.connection.addClient()
+
+ if token is None:
+ return None
+
+ self.transport.set_connection_token(token)
+
+ self.events = uievent.BBUIEventQueue(self.connection, self.clientinfo)
+ for event in bb.event.ui_queue:
+ self.events.queue_event(event)
+
+ _, error = self.connection.runCommand(["setFeatures", self.featureset])
+ if error:
+ # disconnect the client, we can't make the setFeature work
+ self.connection.removeClient()
+ # no need to log it here, the error shall be sent to the client
+ raise BaseException(error)
+
+ return self
+
+ def removeClient(self):
+ if not self.observer_only:
+ self.connection.removeClient()
+
+ def terminate(self):
+ # Don't wait for server indefinitely
+ import socket
+ socket.setdefaulttimeout(2)
+ try:
+ self.events.system_quit()
+ except:
+ pass
+ try:
+ self.connection.removeClient()
+ except:
+ pass
+
+class BitBakeServer(BitBakeBaseServer):
+ def initServer(self, interface = ("localhost", 0)):
+ self.interface = interface
+ self.serverImpl = XMLRPCServer(interface)
+
+ def detach(self):
+ daemonize.createDaemon(self.serverImpl.serve_forever, "bitbake-cookerdaemon.log")
+ del self.cooker
+
+ def establishConnection(self, featureset):
+ self.connection = BitBakeXMLRPCServerConnection(self.serverImpl, self.interface, False, featureset)
+ return self.connection.connect()
+
+ def set_connection_token(self, token):
+ self.connection.transport.set_connection_token(token)
+
+class BitBakeXMLRPCClient(BitBakeBaseServer):
+
+ def __init__(self, observer_only = False, token = None):
+ self.token = token
+
+ self.observer_only = observer_only
+ # if we need extra caches, just tell the server to load them all
+ pass
+
+ def saveConnectionDetails(self, remote):
+ self.remote = remote
+
+ def establishConnection(self, featureset):
+ # The format of "remote" must be "server:port"
+ try:
+ [host, port] = self.remote.split(":")
+ port = int(port)
+ except Exception as e:
+ bb.warn("Failed to read remote definition (%s)" % str(e))
+ raise e
+
+ # We need our IP for the server connection. We get the IP
+ # by trying to connect with the server
+ try:
+ s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+ s.connect((host, port))
+ ip = s.getsockname()[0]
+ s.close()
+ except Exception as e:
+ bb.warn("Could not create socket for %s:%s (%s)" % (host, port, str(e)))
+ raise e
+ try:
+ self.serverImpl = XMLRPCProxyServer(host, port)
+ self.connection = BitBakeXMLRPCServerConnection(self.serverImpl, (ip, 0), self.observer_only, featureset)
+ return self.connection.connect(self.token)
+ except Exception as e:
+ bb.warn("Could not connect to server at %s:%s (%s)" % (host, port, str(e)))
+ raise e
+
+ def endSession(self):
+ self.connection.removeClient()