Squashed 'yocto-poky/' content from commit ea562de
git-subtree-dir: yocto-poky
git-subtree-split: ea562de57590c966cd5a75fda8defecd397e6436
diff --git a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
new file mode 100644
index 0000000..a9909b8
--- /dev/null
+++ b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
@@ -0,0 +1,336 @@
+#
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2014 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import os
+import sys
+import re
+from django.db import transaction
+from django.db.models import Q
+from bldcontrol.models import BuildEnvironment, BRLayer, BRVariable, BRTarget, BRBitbake
+import subprocess
+
+from toastermain import settings
+
+from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException
+
+import logging
+logger = logging.getLogger("toaster")
+
+from pprint import pprint, pformat
+
+class LocalhostBEController(BuildEnvironmentController):
+ """ Implementation of the BuildEnvironmentController for the localhost;
+ this controller manages the default build directory,
+ the server setup and system start and stop for the localhost-type build environment
+
+ """
+
+ def __init__(self, be):
+ super(LocalhostBEController, self).__init__(be)
+ self.dburl = settings.getDATABASE_URL()
+ self.pokydirname = None
+ self.islayerset = False
+
+ def _shellcmd(self, command, cwd = None):
+ if cwd is None:
+ cwd = self.be.sourcedir
+
+ logger.debug("lbc_shellcmmd: (%s) %s" % (cwd, command))
+ p = subprocess.Popen(command, cwd = cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ (out,err) = p.communicate()
+ p.wait()
+ if p.returncode:
+ if len(err) == 0:
+ err = "command: %s \n%s" % (command, out)
+ else:
+ err = "command: %s \n%s" % (command, err)
+ logger.warn("localhostbecontroller: shellcmd error %s" % err)
+ raise ShellCmdException(err)
+ else:
+ logger.debug("localhostbecontroller: shellcmd success")
+ return out
+
+ def _setupBE(self):
+ assert self.pokydirname and os.path.exists(self.pokydirname)
+ path = self.be.builddir
+ if not path:
+ raise Exception("Invalid path creation specified.")
+ if not os.path.exists(path):
+ os.makedirs(path, 0755)
+ self._shellcmd("bash -c \"source %s/oe-init-build-env %s\"" % (self.pokydirname, path))
+ # delete the templateconf.cfg; it may come from an unsupported layer configuration
+ os.remove(os.path.join(path, "conf/templateconf.cfg"))
+
+
+ def writeConfFile(self, file_name, variable_list = None, raw = None):
+ filepath = os.path.join(self.be.builddir, file_name)
+ with open(filepath, "w") as conffile:
+ if variable_list is not None:
+ for i in variable_list:
+ conffile.write("%s=\"%s\"\n" % (i.name, i.value))
+ if raw is not None:
+ conffile.write(raw)
+
+
+ def startBBServer(self):
+ assert self.pokydirname and os.path.exists(self.pokydirname)
+ assert self.islayerset
+
+ # find our own toasterui listener/bitbake
+ from toaster.bldcontrol.management.commands.loadconf import _reduce_canon_path
+
+ own_bitbake = _reduce_canon_path(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../../bin/bitbake"))
+
+ assert os.path.exists(own_bitbake) and os.path.isfile(own_bitbake)
+
+ logger.debug("localhostbecontroller: running the listener at %s" % own_bitbake)
+
+ toaster_ui_log_filepath = os.path.join(self.be.builddir, "toaster_ui.log")
+ # get the file length; we need to detect the _last_ start of the toaster UI, not the first
+ toaster_ui_log_filelength = 0
+ if os.path.exists(toaster_ui_log_filepath):
+ with open(toaster_ui_log_filepath, "w") as f:
+ f.seek(0, 2) # jump to the end
+ toaster_ui_log_filelength = f.tell()
+
+ cmd = "bash -c \"source %s/oe-init-build-env %s 2>&1 >toaster_server.log && bitbake --read %s/conf/toaster-pre.conf --postread %s/conf/toaster.conf --server-only -t xmlrpc -B 0.0.0.0:0 2>&1 >>toaster_server.log \"" % (self.pokydirname, self.be.builddir, self.be.builddir, self.be.builddir)
+
+ port = "-1"
+ logger.debug("localhostbecontroller: starting builder \n%s\n" % cmd)
+
+ cmdoutput = self._shellcmd(cmd)
+ with open(self.be.builddir + "/toaster_server.log", "r") as f:
+ for i in f.readlines():
+ if i.startswith("Bitbake server address"):
+ port = i.split(" ")[-1]
+ logger.debug("localhostbecontroller: Found bitbake server port %s" % port)
+
+ cmd = "bash -c \"source %s/oe-init-build-env-memres -1 %s && DATABASE_URL=%s %s --observe-only -u toasterui --remote-server=0.0.0.0:-1 -t xmlrpc\"" % (self.pokydirname, self.be.builddir, self.dburl, own_bitbake)
+ with open(toaster_ui_log_filepath, "a+") as f:
+ p = subprocess.Popen(cmd, cwd = self.be.builddir, shell=True, stdout=f, stderr=f)
+
+ def _toaster_ui_started(filepath, filepos = 0):
+ if not os.path.exists(filepath):
+ return False
+ with open(filepath, "r") as f:
+ f.seek(filepos)
+ for line in f:
+ if line.startswith("NOTE: ToasterUI waiting for events"):
+ return True
+ return False
+
+ retries = 0
+ started = False
+ while not started and retries < 50:
+ started = _toaster_ui_started(toaster_ui_log_filepath, toaster_ui_log_filelength)
+ import time
+ logger.debug("localhostbecontroller: Waiting bitbake server to start")
+ time.sleep(0.5)
+ retries += 1
+
+ if not started:
+ toaster_ui_log = open(os.path.join(self.be.builddir, "toaster_ui.log"), "r").read()
+ toaster_server_log = open(os.path.join(self.be.builddir, "toaster_server.log"), "r").read()
+ raise BuildSetupException("localhostbecontroller: Bitbake server did not start in 25 seconds, aborting (Error: '%s' '%s')" % (toaster_ui_log, toaster_server_log))
+
+ logger.debug("localhostbecontroller: Started bitbake server")
+
+ while port == "-1":
+ # the port specification is "autodetect"; read the bitbake.lock file
+ with open("%s/bitbake.lock" % self.be.builddir, "r") as f:
+ for line in f.readlines():
+ if ":" in line:
+ port = line.split(":")[1].strip()
+ logger.debug("localhostbecontroller: Autodetected bitbake port %s", port)
+ break
+
+ assert self.be.sourcedir and os.path.exists(self.be.builddir)
+ self.be.bbaddress = "localhost"
+ self.be.bbport = port
+ self.be.bbstate = BuildEnvironment.SERVER_STARTED
+ self.be.save()
+
+ def stopBBServer(self):
+ assert self.pokydirname and os.path.exists(self.pokydirname)
+ assert self.islayerset
+ self._shellcmd("bash -c \"source %s/oe-init-build-env %s && %s source toaster stop\"" %
+ (self.pokydirname, self.be.builddir, (lambda: "" if self.be.bbtoken is None else "BBTOKEN=%s" % self.be.bbtoken)()))
+ self.be.bbstate = BuildEnvironment.SERVER_STOPPED
+ self.be.save()
+ logger.debug("localhostbecontroller: Stopped bitbake server")
+
+ def getGitCloneDirectory(self, url, branch):
+ """ Utility that returns the last component of a git path as directory
+ """
+ import re
+ components = re.split(r'[:\.\/]', url)
+ base = components[-2] if components[-1] == "git" else components[-1]
+
+ if branch != "HEAD":
+ return "_%s_%s.toaster_cloned" % (base, branch)
+
+
+ # word of attention; this is a localhost-specific issue; only on the localhost we expect to have "HEAD" releases
+ # which _ALWAYS_ means the current poky checkout
+ from os.path import dirname as DN
+ local_checkout_path = DN(DN(DN(DN(DN(os.path.abspath(__file__))))))
+ #logger.debug("localhostbecontroller: using HEAD checkout in %s" % local_checkout_path)
+ return local_checkout_path
+
+
+ def setLayers(self, bitbakes, layers):
+ """ a word of attention: by convention, the first layer for any build will be poky! """
+
+ assert self.be.sourcedir is not None
+ assert len(bitbakes) == 1
+ # set layers in the layersource
+
+ # 1. get a list of repos with branches, and map dirpaths for each layer
+ gitrepos = {}
+
+ gitrepos[(bitbakes[0].giturl, bitbakes[0].commit)] = []
+ gitrepos[(bitbakes[0].giturl, bitbakes[0].commit)].append( ("bitbake", bitbakes[0].dirpath) )
+
+ for layer in layers:
+ # we don't process local URLs
+ if layer.giturl.startswith("file://"):
+ continue
+ if not (layer.giturl, layer.commit) in gitrepos:
+ gitrepos[(layer.giturl, layer.commit)] = []
+ gitrepos[(layer.giturl, layer.commit)].append( (layer.name, layer.dirpath) )
+
+
+ logger.debug("localhostbecontroller, our git repos are %s" % pformat(gitrepos))
+
+
+ # 2. find checked-out git repos in the sourcedir directory that may help faster cloning
+
+ cached_layers = {}
+ for ldir in os.listdir(self.be.sourcedir):
+ fldir = os.path.join(self.be.sourcedir, ldir)
+ if os.path.isdir(fldir):
+ try:
+ for line in self._shellcmd("git remote -v", fldir).split("\n"):
+ try:
+ remote = line.split("\t")[1].split(" ")[0]
+ if remote not in cached_layers:
+ cached_layers[remote] = fldir
+ except IndexError:
+ pass
+ except ShellCmdException:
+ # ignore any errors in collecting git remotes
+ pass
+
+ layerlist = []
+
+
+ # 3. checkout the repositories
+ for giturl, commit in gitrepos.keys():
+ localdirname = os.path.join(self.be.sourcedir, self.getGitCloneDirectory(giturl, commit))
+ logger.debug("localhostbecontroller: giturl %s:%s checking out in current directory %s" % (giturl, commit, localdirname))
+
+ # make sure our directory is a git repository
+ if os.path.exists(localdirname):
+ localremotes = self._shellcmd("git remote -v", localdirname)
+ if not giturl in localremotes:
+ raise BuildSetupException("Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something." % (localdirname, ", ".join(localremotes.split("\n")), giturl))
+ else:
+ if giturl in cached_layers:
+ logger.debug("localhostbecontroller git-copying %s to %s" % (cached_layers[giturl], localdirname))
+ self._shellcmd("git clone \"%s\" \"%s\"" % (cached_layers[giturl], localdirname))
+ self._shellcmd("git remote remove origin", localdirname)
+ self._shellcmd("git remote add origin \"%s\"" % giturl, localdirname)
+ else:
+ logger.debug("localhostbecontroller: cloning %s:%s in %s" % (giturl, commit, localdirname))
+ self._shellcmd("git clone \"%s\" --single-branch --branch \"%s\" \"%s\"" % (giturl, commit, localdirname))
+
+ # branch magic name "HEAD" will inhibit checkout
+ if commit != "HEAD":
+ logger.debug("localhostbecontroller: checking out commit %s to %s " % (commit, localdirname))
+ self._shellcmd("git fetch --all && git checkout \"%s\" && git rebase \"origin/%s\"" % (commit, commit) , localdirname)
+
+ # take the localdirname as poky dir if we can find the oe-init-build-env
+ if self.pokydirname is None and os.path.exists(os.path.join(localdirname, "oe-init-build-env")):
+ logger.debug("localhostbecontroller: selected poky dir name %s" % localdirname)
+ self.pokydirname = localdirname
+
+ # make sure we have a working bitbake
+ if not os.path.exists(os.path.join(self.pokydirname, 'bitbake')):
+ logger.debug("localhostbecontroller: checking bitbake into the poky dirname %s " % self.pokydirname)
+ self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname, 'bitbake')))
+
+ # verify our repositories
+ for name, dirpath in gitrepos[(giturl, commit)]:
+ localdirpath = os.path.join(localdirname, dirpath)
+ logger.debug("localhostbecontroller: localdirpath expected '%s'" % localdirpath)
+ if not os.path.exists(localdirpath):
+ raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit))
+
+ if name != "bitbake":
+ layerlist.append(localdirpath.rstrip("/"))
+
+ logger.debug("localhostbecontroller: current layer list %s " % pformat(layerlist))
+
+ # 4. configure the build environment, so we have a conf/bblayers.conf
+ assert self.pokydirname is not None
+ self._setupBE()
+
+ # 5. update the bblayers.conf
+ bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf")
+ if not os.path.exists(bblayerconf):
+ raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf)
+
+ BuildEnvironmentController._updateBBLayers(bblayerconf, layerlist)
+
+ self.islayerset = True
+ return True
+
+ def readServerLogFile(self):
+ return open(os.path.join(self.be.builddir, "toaster_server.log"), "r").read()
+
+ def release(self):
+ assert self.be.sourcedir and os.path.exists(self.be.builddir)
+ import shutil
+ shutil.rmtree(os.path.join(self.be.sourcedir, "build"))
+ assert not os.path.exists(self.be.builddir)
+
+
+ def triggerBuild(self, bitbake, layers, variables, targets):
+ # set up the buid environment with the needed layers
+ self.setLayers(bitbake, layers)
+ self.writeConfFile("conf/toaster-pre.conf", variables)
+ self.writeConfFile("conf/toaster.conf", raw = "INHERIT+=\"toaster buildhistory\"")
+
+ # get the bb server running with the build req id and build env id
+ bbctrl = self.getBBController()
+
+ # trigger the build command
+ task = reduce(lambda x, y: x if len(y)== 0 else y, map(lambda y: y.task, targets))
+ if len(task) == 0:
+ task = None
+
+ bbctrl.build(list(map(lambda x:x.target, targets)), task)
+
+ logger.debug("localhostbecontroller: Build launched, exiting. Follow build logs at %s/toaster_ui.log" % self.be.builddir)
+
+ # disconnect from the server
+ bbctrl.disconnect()