| # |
| # ex:ts=4:sw=4:sts=4:et |
| # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- |
| # |
| # BitBake Toaster Implementation |
| # |
| # Copyright (C) 2014 Intel Corporation |
| # |
| # This program is free software; you can redistribute it and/or modify |
| # it under the terms of the GNU General Public License version 2 as |
| # published by the Free Software Foundation. |
| # |
| # This program is distributed in the hope that it will be useful, |
| # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| # GNU General Public License for more details. |
| # |
| # You should have received a copy of the GNU General Public License along |
| # with this program; if not, write to the Free Software Foundation, Inc., |
| # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. |
| |
| |
| import os |
| import sys |
| import re |
| import shutil |
| from django.db import transaction |
| from django.db.models import Q |
| from bldcontrol.models import BuildEnvironment, BRLayer, BRVariable, BRTarget, BRBitbake |
| from orm.models import CustomImageRecipe, Layer, Layer_Version, ProjectLayer |
| import subprocess |
| |
| from toastermain import settings |
| |
| from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException |
| |
| import logging |
| logger = logging.getLogger("toaster") |
| |
| from pprint import pprint, pformat |
| |
| class LocalhostBEController(BuildEnvironmentController): |
| """ Implementation of the BuildEnvironmentController for the localhost; |
| this controller manages the default build directory, |
| the server setup and system start and stop for the localhost-type build environment |
| |
| """ |
| |
| def __init__(self, be): |
| super(LocalhostBEController, self).__init__(be) |
| self.dburl = settings.getDATABASE_URL() |
| self.pokydirname = None |
| self.islayerset = False |
| |
| def _shellcmd(self, command, cwd = None): |
| if cwd is None: |
| cwd = self.be.sourcedir |
| |
| logger.debug("lbc_shellcmmd: (%s) %s" % (cwd, command)) |
| p = subprocess.Popen(command, cwd = cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) |
| (out,err) = p.communicate() |
| p.wait() |
| if p.returncode: |
| if len(err) == 0: |
| err = "command: %s \n%s" % (command, out) |
| else: |
| err = "command: %s \n%s" % (command, err) |
| logger.warn("localhostbecontroller: shellcmd error %s" % err) |
| raise ShellCmdException(err) |
| else: |
| logger.debug("localhostbecontroller: shellcmd success") |
| return out |
| |
| def _setupBE(self): |
| assert self.pokydirname and os.path.exists(self.pokydirname) |
| path = self.be.builddir |
| if not path: |
| raise Exception("Invalid path creation specified.") |
| if not os.path.exists(path): |
| os.makedirs(path, 0755) |
| self._shellcmd("bash -c \"source %s/oe-init-build-env %s\"" % (self.pokydirname, path)) |
| # delete the templateconf.cfg; it may come from an unsupported layer configuration |
| os.remove(os.path.join(path, "conf/templateconf.cfg")) |
| |
| |
| def writeConfFile(self, file_name, variable_list = None, raw = None): |
| filepath = os.path.join(self.be.builddir, file_name) |
| with open(filepath, "w") as conffile: |
| if variable_list is not None: |
| for i in variable_list: |
| conffile.write("%s=\"%s\"\n" % (i.name, i.value)) |
| if raw is not None: |
| conffile.write(raw) |
| |
| |
| def startBBServer(self): |
| assert self.pokydirname and os.path.exists(self.pokydirname) |
| assert self.islayerset |
| |
| # find our own toasterui listener/bitbake |
| from toaster.bldcontrol.management.commands.loadconf import _reduce_canon_path |
| |
| own_bitbake = _reduce_canon_path(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../../bin/bitbake")) |
| |
| assert os.path.exists(own_bitbake) and os.path.isfile(own_bitbake) |
| |
| logger.debug("localhostbecontroller: running the listener at %s" % own_bitbake) |
| |
| toaster_ui_log_filepath = os.path.join(self.be.builddir, "toaster_ui.log") |
| # get the file length; we need to detect the _last_ start of the toaster UI, not the first |
| toaster_ui_log_filelength = 0 |
| if os.path.exists(toaster_ui_log_filepath): |
| with open(toaster_ui_log_filepath, "w") as f: |
| f.seek(0, 2) # jump to the end |
| toaster_ui_log_filelength = f.tell() |
| |
| cmd = "bash -c \"source %s/oe-init-build-env %s 2>&1 >toaster_server.log && bitbake --read %s/conf/toaster-pre.conf --postread %s/conf/toaster.conf --server-only -t xmlrpc -B 0.0.0.0:0 2>&1 >>toaster_server.log \"" % (self.pokydirname, self.be.builddir, self.be.builddir, self.be.builddir) |
| |
| port = "-1" |
| logger.debug("localhostbecontroller: starting builder \n%s\n" % cmd) |
| |
| cmdoutput = self._shellcmd(cmd) |
| with open(self.be.builddir + "/toaster_server.log", "r") as f: |
| for i in f.readlines(): |
| if i.startswith("Bitbake server address"): |
| port = i.split(" ")[-1] |
| logger.debug("localhostbecontroller: Found bitbake server port %s" % port) |
| |
| cmd = "bash -c \"source %s/oe-init-build-env-memres -1 %s && DATABASE_URL=%s %s --observe-only -u toasterui --remote-server=0.0.0.0:-1 -t xmlrpc\"" % (self.pokydirname, self.be.builddir, self.dburl, own_bitbake) |
| with open(toaster_ui_log_filepath, "a+") as f: |
| p = subprocess.Popen(cmd, cwd = self.be.builddir, shell=True, stdout=f, stderr=f) |
| |
| def _toaster_ui_started(filepath, filepos = 0): |
| if not os.path.exists(filepath): |
| return False |
| with open(filepath, "r") as f: |
| f.seek(filepos) |
| for line in f: |
| if line.startswith("NOTE: ToasterUI waiting for events"): |
| return True |
| return False |
| |
| retries = 0 |
| started = False |
| while not started and retries < 50: |
| started = _toaster_ui_started(toaster_ui_log_filepath, toaster_ui_log_filelength) |
| import time |
| logger.debug("localhostbecontroller: Waiting bitbake server to start") |
| time.sleep(0.5) |
| retries += 1 |
| |
| if not started: |
| toaster_ui_log = open(os.path.join(self.be.builddir, "toaster_ui.log"), "r").read() |
| toaster_server_log = open(os.path.join(self.be.builddir, "toaster_server.log"), "r").read() |
| raise BuildSetupException("localhostbecontroller: Bitbake server did not start in 25 seconds, aborting (Error: '%s' '%s')" % (toaster_ui_log, toaster_server_log)) |
| |
| logger.debug("localhostbecontroller: Started bitbake server") |
| |
| while port == "-1": |
| # the port specification is "autodetect"; read the bitbake.lock file |
| with open("%s/bitbake.lock" % self.be.builddir, "r") as f: |
| for line in f.readlines(): |
| if ":" in line: |
| port = line.split(":")[1].strip() |
| logger.debug("localhostbecontroller: Autodetected bitbake port %s", port) |
| break |
| |
| assert self.be.sourcedir and os.path.exists(self.be.builddir) |
| self.be.bbaddress = "localhost" |
| self.be.bbport = port |
| self.be.bbstate = BuildEnvironment.SERVER_STARTED |
| self.be.save() |
| |
| def stopBBServer(self): |
| assert self.pokydirname and os.path.exists(self.pokydirname) |
| assert self.islayerset |
| self._shellcmd("bash -c \"source %s/oe-init-build-env %s && %s source toaster stop\"" % |
| (self.pokydirname, self.be.builddir, (lambda: "" if self.be.bbtoken is None else "BBTOKEN=%s" % self.be.bbtoken)())) |
| self.be.bbstate = BuildEnvironment.SERVER_STOPPED |
| self.be.save() |
| logger.debug("localhostbecontroller: Stopped bitbake server") |
| |
| def getGitCloneDirectory(self, url, branch): |
| """Construct unique clone directory name out of url and branch.""" |
| if branch != "HEAD": |
| return "_toaster_clones/_%s_%s" % (re.sub('[:/@%]', '_', url), branch) |
| |
| # word of attention; this is a localhost-specific issue; only on the localhost we expect to have "HEAD" releases |
| # which _ALWAYS_ means the current poky checkout |
| from os.path import dirname as DN |
| local_checkout_path = DN(DN(DN(DN(DN(os.path.abspath(__file__)))))) |
| #logger.debug("localhostbecontroller: using HEAD checkout in %s" % local_checkout_path) |
| return local_checkout_path |
| |
| |
| def setLayers(self, bitbakes, layers, targets): |
| """ a word of attention: by convention, the first layer for any build will be poky! """ |
| |
| assert self.be.sourcedir is not None |
| assert len(bitbakes) == 1 |
| # set layers in the layersource |
| |
| # 1. get a list of repos with branches, and map dirpaths for each layer |
| gitrepos = {} |
| |
| gitrepos[(bitbakes[0].giturl, bitbakes[0].commit)] = [] |
| gitrepos[(bitbakes[0].giturl, bitbakes[0].commit)].append( ("bitbake", bitbakes[0].dirpath) ) |
| |
| for layer in layers: |
| # we don't process local URLs |
| if layer.giturl.startswith("file://"): |
| continue |
| if not (layer.giturl, layer.commit) in gitrepos: |
| gitrepos[(layer.giturl, layer.commit)] = [] |
| gitrepos[(layer.giturl, layer.commit)].append( (layer.name, layer.dirpath) ) |
| |
| |
| logger.debug("localhostbecontroller, our git repos are %s" % pformat(gitrepos)) |
| |
| |
| # 2. Note for future use if the current source directory is a |
| # checked-out git repos that could match a layer's vcs_url and therefore |
| # be used to speed up cloning (rather than fetching it again). |
| |
| cached_layers = {} |
| |
| try: |
| for remotes in self._shellcmd("git remote -v", self.be.sourcedir).split("\n"): |
| try: |
| remote = remotes.split("\t")[1].split(" ")[0] |
| if remote not in cached_layers: |
| cached_layers[remote] = self.be.sourcedir |
| except IndexError: |
| pass |
| except ShellCmdException: |
| # ignore any errors in collecting git remotes this is an optional |
| # step |
| pass |
| |
| logger.info("Using pre-checked out source for layer %s", cached_layers) |
| |
| layerlist = [] |
| |
| |
| # 3. checkout the repositories |
| for giturl, commit in gitrepos.keys(): |
| localdirname = os.path.join(self.be.sourcedir, self.getGitCloneDirectory(giturl, commit)) |
| logger.debug("localhostbecontroller: giturl %s:%s checking out in current directory %s" % (giturl, commit, localdirname)) |
| |
| # make sure our directory is a git repository |
| if os.path.exists(localdirname): |
| localremotes = self._shellcmd("git remote -v", localdirname) |
| if not giturl in localremotes: |
| raise BuildSetupException("Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something." % (localdirname, ", ".join(localremotes.split("\n")), giturl)) |
| else: |
| if giturl in cached_layers: |
| logger.debug("localhostbecontroller git-copying %s to %s" % (cached_layers[giturl], localdirname)) |
| self._shellcmd("git clone \"%s\" \"%s\"" % (cached_layers[giturl], localdirname)) |
| self._shellcmd("git remote remove origin", localdirname) |
| self._shellcmd("git remote add origin \"%s\"" % giturl, localdirname) |
| else: |
| logger.debug("localhostbecontroller: cloning %s in %s" % (giturl, localdirname)) |
| self._shellcmd('git clone "%s" "%s"' % (giturl, localdirname)) |
| |
| # branch magic name "HEAD" will inhibit checkout |
| if commit != "HEAD": |
| logger.debug("localhostbecontroller: checking out commit %s to %s " % (commit, localdirname)) |
| ref = commit if re.match('^[a-fA-F0-9]+$', commit) else 'origin/%s' % commit |
| self._shellcmd('git fetch --all && git reset --hard "%s"' % ref, localdirname) |
| |
| # take the localdirname as poky dir if we can find the oe-init-build-env |
| if self.pokydirname is None and os.path.exists(os.path.join(localdirname, "oe-init-build-env")): |
| logger.debug("localhostbecontroller: selected poky dir name %s" % localdirname) |
| self.pokydirname = localdirname |
| |
| # make sure we have a working bitbake |
| if not os.path.exists(os.path.join(self.pokydirname, 'bitbake')): |
| logger.debug("localhostbecontroller: checking bitbake into the poky dirname %s " % self.pokydirname) |
| self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname, 'bitbake'))) |
| |
| # verify our repositories |
| for name, dirpath in gitrepos[(giturl, commit)]: |
| localdirpath = os.path.join(localdirname, dirpath) |
| logger.debug("localhostbecontroller: localdirpath expected '%s'" % localdirpath) |
| if not os.path.exists(localdirpath): |
| raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit)) |
| |
| if name != "bitbake": |
| layerlist.append(localdirpath.rstrip("/")) |
| |
| logger.debug("localhostbecontroller: current layer list %s " % pformat(layerlist)) |
| |
| # 4. configure the build environment, so we have a conf/bblayers.conf |
| assert self.pokydirname is not None |
| self._setupBE() |
| |
| # 5. update the bblayers.conf |
| bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf") |
| if not os.path.exists(bblayerconf): |
| raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf) |
| |
| # 6. create custom layer and add custom recipes to it |
| layerpath = os.path.join(self.be.sourcedir, "_meta-toaster-custom") |
| if os.path.isdir(layerpath): |
| shutil.rmtree(layerpath) # remove leftovers from previous builds |
| for target in targets: |
| try: |
| customrecipe = CustomImageRecipe.objects.get(name=target.target, |
| project=bitbakes[0].req.project) |
| except CustomImageRecipe.DoesNotExist: |
| continue # not a custom recipe, skip |
| |
| # create directory structure |
| for name in ("conf", "recipes"): |
| path = os.path.join(layerpath, name) |
| if not os.path.isdir(path): |
| os.makedirs(path) |
| |
| # create layer.oonf |
| config = os.path.join(layerpath, "conf", "layer.conf") |
| if not os.path.isfile(config): |
| with open(config, "w") as conf: |
| conf.write('BBPATH .= ":${LAYERDIR}"\nBBFILES += "${LAYERDIR}/recipes/*.bb"\n') |
| |
| # create recipe |
| recipe = os.path.join(layerpath, "recipes", "%s.bb" % target.target) |
| with open(recipe, "w") as recipef: |
| recipef.write("require %s\n" % customrecipe.base_recipe.recipe.file_path) |
| packages = [pkg.name for pkg in customrecipe.packages.all()] |
| if packages: |
| recipef.write('IMAGE_INSTALL = "%s"\n' % ' '.join(packages)) |
| |
| # create *Layer* objects needed for build machinery to work |
| layer = Layer.objects.get_or_create(name="Toaster Custom layer", |
| summary="Layer for custom recipes", |
| vcs_url="file://%s" % layerpath)[0] |
| breq = target.req |
| lver = Layer_Version.objects.get_or_create(project=breq.project, layer=layer, |
| dirpath=layerpath, build=breq.build)[0] |
| ProjectLayer.objects.get_or_create(project=breq.project, layercommit=lver, |
| optional=False) |
| BRLayer.objects.get_or_create(req=breq, name=layer.name, dirpath=layerpath, |
| giturl="file://%s" % layerpath) |
| if os.path.isdir(layerpath): |
| layerlist.append(layerpath) |
| |
| BuildEnvironmentController._updateBBLayers(bblayerconf, layerlist) |
| |
| self.islayerset = True |
| return True |
| |
| def readServerLogFile(self): |
| return open(os.path.join(self.be.builddir, "toaster_server.log"), "r").read() |
| |
| def release(self): |
| assert self.be.sourcedir and os.path.exists(self.be.builddir) |
| import shutil |
| shutil.rmtree(os.path.join(self.be.sourcedir, "build")) |
| assert not os.path.exists(self.be.builddir) |
| |
| |
| def triggerBuild(self, bitbake, layers, variables, targets): |
| # set up the buid environment with the needed layers |
| self.setLayers(bitbake, layers, targets) |
| self.writeConfFile("conf/toaster-pre.conf", variables) |
| self.writeConfFile("conf/toaster.conf", raw = "INHERIT+=\"toaster buildhistory\"") |
| |
| # get the bb server running with the build req id and build env id |
| bbctrl = self.getBBController() |
| |
| # trigger the build command |
| task = reduce(lambda x, y: x if len(y)== 0 else y, map(lambda y: y.task, targets)) |
| if len(task) == 0: |
| task = None |
| |
| bbctrl.build(list(map(lambda x:x.target, targets)), task) |
| |
| logger.debug("localhostbecontroller: Build launched, exiting. Follow build logs at %s/toaster_ui.log" % self.be.builddir) |
| |
| # disconnect from the server |
| bbctrl.disconnect() |