Yocto 2.4
Move OpenBMC to Yocto 2.4(rocko)
Tested: Built and verified Witherspoon and Palmetto images
Change-Id: I12057b18610d6fb0e6903c60213690301e9b0c67
Signed-off-by: Brad Bishop <bradleyb@fuzziesquirrel.com>
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcollector/urls.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcollector/urls.py
index 64722f2..888175d 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/bldcollector/urls.py
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcollector/urls.py
@@ -1,7 +1,7 @@
#
# BitBake Toaster Implementation
#
-# Copyright (C) 2014 Intel Corporation
+# Copyright (C) 2014-2017 Intel Corporation
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@@ -17,9 +17,11 @@
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-from django.conf.urls import patterns, include, url
+from django.conf.urls import include, url
-urlpatterns = patterns('bldcollector.views',
+import bldcollector.views
+
+urlpatterns = [
# landing point for pushing a bitbake_eventlog.json file to this toaster instace
- url(r'^eventfile$', 'eventfile', name='eventfile'),
- )
+ url(r'^eventfile$', bldcollector.views.eventfile, name='eventfile'),
+]
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/bbcontroller.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/bbcontroller.py
index 912f67b..5195600 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/bbcontroller.py
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/bbcontroller.py
@@ -37,8 +37,8 @@
"""
def __init__(self, be):
- import bb.server.xmlrpc
- self.connection = bb.server.xmlrpc._create_server(be.bbaddress,
+ import bb.server.xmlrpcclient
+ self.connection = bb.server.xmlrpcclient._create_server(be.bbaddress,
int(be.bbport))[0]
def _runCommand(self, command):
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
index 1207102..4c17562 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
@@ -24,6 +24,7 @@
import sys
import re
import shutil
+import time
from django.db import transaction
from django.db.models import Q
from bldcontrol.models import BuildEnvironment, BRLayer, BRVariable, BRTarget, BRBitbake
@@ -51,12 +52,14 @@
self.pokydirname = None
self.islayerset = False
- def _shellcmd(self, command, cwd=None, nowait=False):
+ def _shellcmd(self, command, cwd=None, nowait=False,env=None):
if cwd is None:
cwd = self.be.sourcedir
+ if env is None:
+ env=os.environ.copy()
- logger.debug("lbc_shellcmmd: (%s) %s" % (cwd, command))
- p = subprocess.Popen(command, cwd = cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ logger.debug("lbc_shellcmd: (%s) %s" % (cwd, command))
+ p = subprocess.Popen(command, cwd = cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
if nowait:
return
(out,err) = p.communicate()
@@ -85,6 +88,11 @@
return local_checkout_path
+ def setCloneStatus(self,bitbake,status,total,current):
+ bitbake.req.build.repos_cloned=current
+ bitbake.req.build.repos_to_clone=total
+ bitbake.req.build.save()
+
def setLayers(self, bitbake, layers, targets):
""" a word of attention: by convention, the first layer for any build will be poky! """
@@ -92,6 +100,8 @@
layerlist = []
nongitlayerlist = []
+ git_env = os.environ.copy()
+ # (note: add custom environment settings here)
# set layers in the layersource
@@ -132,7 +142,7 @@
cached_layers = {}
try:
- for remotes in self._shellcmd("git remote -v", self.be.sourcedir).split("\n"):
+ for remotes in self._shellcmd("git remote -v", self.be.sourcedir,env=git_env).split("\n"):
try:
remote = remotes.split("\t")[1].split(" ")[0]
if remote not in cached_layers:
@@ -147,7 +157,13 @@
logger.info("Using pre-checked out source for layer %s", cached_layers)
# 3. checkout the repositories
+ clone_count=0
+ clone_total=len(gitrepos.keys())
+ self.setCloneStatus(bitbake,'Started',clone_total,clone_count)
for giturl, commit in gitrepos.keys():
+ self.setCloneStatus(bitbake,'progress',clone_total,clone_count)
+ clone_count += 1
+
localdirname = os.path.join(self.be.sourcedir, self.getGitCloneDirectory(giturl, commit))
logger.debug("localhostbecontroller: giturl %s:%s checking out in current directory %s" % (giturl, commit, localdirname))
@@ -155,7 +171,7 @@
if os.path.exists(localdirname):
try:
localremotes = self._shellcmd("git remote -v",
- localdirname)
+ localdirname,env=git_env)
if not giturl in localremotes and commit != 'HEAD':
raise BuildSetupException("Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something." % (localdirname, ", ".join(localremotes.split("\n")), giturl))
except ShellCmdException:
@@ -165,18 +181,18 @@
else:
if giturl in cached_layers:
logger.debug("localhostbecontroller git-copying %s to %s" % (cached_layers[giturl], localdirname))
- self._shellcmd("git clone \"%s\" \"%s\"" % (cached_layers[giturl], localdirname))
- self._shellcmd("git remote remove origin", localdirname)
- self._shellcmd("git remote add origin \"%s\"" % giturl, localdirname)
+ self._shellcmd("git clone \"%s\" \"%s\"" % (cached_layers[giturl], localdirname),env=git_env)
+ self._shellcmd("git remote remove origin", localdirname,env=git_env)
+ self._shellcmd("git remote add origin \"%s\"" % giturl, localdirname,env=git_env)
else:
logger.debug("localhostbecontroller: cloning %s in %s" % (giturl, localdirname))
- self._shellcmd('git clone "%s" "%s"' % (giturl, localdirname))
+ self._shellcmd('git clone "%s" "%s"' % (giturl, localdirname),env=git_env)
# branch magic name "HEAD" will inhibit checkout
if commit != "HEAD":
logger.debug("localhostbecontroller: checking out commit %s to %s " % (commit, localdirname))
ref = commit if re.match('^[a-fA-F0-9]+$', commit) else 'origin/%s' % commit
- self._shellcmd('git fetch --all && git reset --hard "%s"' % ref, localdirname)
+ self._shellcmd('git fetch --all && git reset --hard "%s"' % ref, localdirname,env=git_env)
# take the localdirname as poky dir if we can find the oe-init-build-env
if self.pokydirname is None and os.path.exists(os.path.join(localdirname, "oe-init-build-env")):
@@ -186,7 +202,7 @@
# make sure we have a working bitbake
if not os.path.exists(os.path.join(self.pokydirname, 'bitbake')):
logger.debug("localhostbecontroller: checking bitbake into the poky dirname %s " % self.pokydirname)
- self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbake.commit, bitbake.giturl, os.path.join(self.pokydirname, 'bitbake')))
+ self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbake.commit, bitbake.giturl, os.path.join(self.pokydirname, 'bitbake')),env=git_env)
# verify our repositories
for name, dirpath in gitrepos[(giturl, commit)]:
@@ -198,6 +214,7 @@
if name != "bitbake":
layerlist.append(localdirpath.rstrip("/"))
+ self.setCloneStatus(bitbake,'complete',clone_total,clone_count)
logger.debug("localhostbecontroller: current layer list %s " % pformat(layerlist))
if self.pokydirname is None and os.path.exists(os.path.join(self.be.sourcedir, "oe-init-build-env")):
@@ -319,16 +336,32 @@
conf.write('%s="%s"\n' % (var.name, var.value))
conf.write('INHERIT+="toaster buildhistory"')
+ # clean the Toaster to build environment
+ env_clean = 'unset BBPATH;' # clean BBPATH for <= YP-2.4.0
+
# run bitbake server from the clone
bitbake = os.path.join(self.pokydirname, 'bitbake', 'bin', 'bitbake')
toasterlayers = os.path.join(builddir,"conf/toaster-bblayers.conf")
- self._shellcmd('bash -c \"source %s %s; BITBAKE_UI="knotty" %s --read %s --read %s '
- '--server-only -t xmlrpc -B 0.0.0.0:0\"' % (oe_init,
+ self._shellcmd('%s bash -c \"source %s %s; BITBAKE_UI="knotty" %s --read %s --read %s '
+ '--server-only -B 0.0.0.0:0\"' % (env_clean, oe_init,
builddir, bitbake, confpath, toasterlayers), self.be.sourcedir)
# read port number from bitbake.lock
- self.be.bbport = ""
+ self.be.bbport = -1
bblock = os.path.join(builddir, 'bitbake.lock')
+ # allow 10 seconds for bb lock file to appear but also be populated
+ for lock_check in range(10):
+ if not os.path.exists(bblock):
+ logger.debug("localhostbecontroller: waiting for bblock file to appear")
+ time.sleep(1)
+ continue
+ if 10 < os.stat(bblock).st_size:
+ break
+ logger.debug("localhostbecontroller: waiting for bblock content to appear")
+ time.sleep(1)
+ else:
+ raise BuildSetupException("Cannot find bitbake server lock file '%s'. Aborting." % bblock)
+
with open(bblock) as fplock:
for line in fplock:
if ":" in line:
@@ -336,7 +369,7 @@
logger.debug("localhostbecontroller: bitbake port %s", self.be.bbport)
break
- if not self.be.bbport:
+ if -1 == self.be.bbport:
raise BuildSetupException("localhostbecontroller: can't read bitbake port from %s" % bblock)
self.be.bbaddress = "localhost"
@@ -357,10 +390,11 @@
log = os.path.join(builddir, 'toaster_ui.log')
local_bitbake = os.path.join(os.path.dirname(os.getenv('BBBASEDIR')),
'bitbake')
- self._shellcmd(['bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:-1" '
- '%s %s -u toasterui --token="" >>%s 2>&1;'
- 'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:-1 %s -m)&\"' \
- % (brbe, local_bitbake, bbtargets, log, bitbake)],
+ self._shellcmd(['%s bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:%s" '
+ '%s %s -u toasterui --read %s --read %s --token="" >>%s 2>&1;'
+ 'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:%s %s -m)&\"' \
+ % (env_clean, brbe, self.be.bbport, local_bitbake, bbtargets, confpath, toasterlayers, log,
+ self.be.bbport, bitbake,)],
builddir, nowait=True)
logger.debug('localhostbecontroller: Build launched, exiting. '
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py
index 2ed994f..582114a 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py
@@ -1,4 +1,4 @@
-from django.core.management.base import NoArgsCommand, CommandError
+from django.core.management.base import BaseCommand, CommandError
from django.db import transaction
from django.core.management import call_command
@@ -18,7 +18,7 @@
return os.path.dirname(path)
-class Command(NoArgsCommand):
+class Command(BaseCommand):
args = ""
help = "Verifies that the configured settings are valid and usable, or prompts the user to fix the settings."
@@ -75,7 +75,10 @@
call_command("loaddata", "settings")
template_conf = os.environ.get("TEMPLATECONF", "")
- if "poky" in template_conf:
+ if ToasterSetting.objects.filter(name='CUSTOM_XML_ONLY').count() > 0:
+ # only use the custom settings
+ pass
+ elif "poky" in template_conf:
print("Loading poky configuration")
call_command("loaddata", "poky")
else:
@@ -152,7 +155,7 @@
- def handle_noargs(self, **options):
+ def handle(self, **options):
retval = 0
retval += self._verify_build_environment()
retval += self._verify_default_settings()
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py
index df11f9d..791e53e 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py
@@ -1,4 +1,4 @@
-from django.core.management.base import NoArgsCommand
+from django.core.management.base import BaseCommand
from django.db import transaction
from django.db.models import Q
@@ -16,7 +16,7 @@
logger = logging.getLogger("toaster")
-class Command(NoArgsCommand):
+class Command(BaseCommand):
args = ""
help = "Schedules and executes build requests as possible. "\
"Does not return (interrupt with Ctrl-C)"
@@ -79,6 +79,14 @@
br.save()
bec.be.lock = BuildEnvironment.LOCK_FREE
bec.be.save()
+ # Cancel the pending build and report the exception to the UI
+ log_object = LogMessage.objects.create(
+ build = br.build,
+ level = LogMessage.EXCEPTION,
+ message = errmsg)
+ log_object.save()
+ br.build.outcome = Build.FAILED
+ br.build.save()
def archive(self):
for br in BuildRequest.objects.filter(state=BuildRequest.REQ_ARCHIVE):
@@ -168,7 +176,7 @@
except Exception as e:
logger.warn("runbuilds: schedule exception %s" % str(e))
- def handle_noargs(self, **options):
+ def handle(self, **options):
pidfile_path = os.path.join(os.environ.get("BUILDDIR", "."),
".runbuilds.pid")
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/orm/fixtures/custom_toaster_append.sh_sample b/import-layers/yocto-poky/bitbake/lib/toaster/orm/fixtures/custom_toaster_append.sh_sample
new file mode 100755
index 0000000..8c4e163
--- /dev/null
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/orm/fixtures/custom_toaster_append.sh_sample
@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# Copyright (C) 2017 Intel Corp.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+# This is sample software. Rename it to 'custom_toaster_append.sh' and
+# enable the respective custom sections.
+
+verbose=0
+if [ $verbose -ne 0 ] ; then
+ echo "custom_toaster_append.sh:$*"
+fi
+
+if [ "toaster_prepend" = "$1" ] ; then
+ echo "Add custom actions here when Toaster script is started"
+fi
+
+if [ "web_start_postpend" = "$1" ] ; then
+ echo "Add custom actions here after Toaster web service is started"
+fi
+
+if [ "web_stop_postpend" = "$1" ] ; then
+ echo "Add custom actions here after Toaster web service is stopped"
+fi
+
+if [ "noweb_start_postpend" = "$1" ] ; then
+ echo "Add custom actions here after Toaster (no web) service is started"
+fi
+
+if [ "noweb_stop_postpend" = "$1" ] ; then
+ echo "Add custom actions here after Toaster (no web) service is stopped"
+fi
+
+if [ "toaster_postpend" = "$1" ] ; then
+ echo "Add custom actions here after Toaster script is done"
+fi
+
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/orm/fixtures/oe-core.xml b/import-layers/yocto-poky/bitbake/lib/toaster/orm/fixtures/oe-core.xml
index 66c3595..00720c3 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/orm/fixtures/oe-core.xml
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/orm/fixtures/oe-core.xml
@@ -8,9 +8,9 @@
<!-- Bitbake versions which correspond to the metadata release -->
<object model="orm.bitbakeversion" pk="1">
- <field type="CharField" name="name">pyro</field>
+ <field type="CharField" name="name">rocko</field>
<field type="CharField" name="giturl">git://git.openembedded.org/bitbake</field>
- <field type="CharField" name="branch">1.34</field>
+ <field type="CharField" name="branch">1.36</field>
</object>
<object model="orm.bitbakeversion" pk="2">
<field type="CharField" name="name">HEAD</field>
@@ -25,11 +25,11 @@
<!-- Releases available -->
<object model="orm.release" pk="1">
- <field type="CharField" name="name">pyro</field>
- <field type="CharField" name="description">Openembedded Pyro</field>
+ <field type="CharField" name="name">rocko</field>
+ <field type="CharField" name="description">Openembedded Rocko</field>
<field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">1</field>
- <field type="CharField" name="branch_name">pyro</field>
- <field type="TextField" name="helptext">Toaster will run your builds using the tip of the <a href=\"http://cgit.openembedded.org/openembedded-core/log/?h=pyro\">OpenEmbedded Pyro</a> branch.</field>
+ <field type="CharField" name="branch_name">rocko</field>
+ <field type="TextField" name="helptext">Toaster will run your builds using the tip of the <a href=\"http://cgit.openembedded.org/openembedded-core/log/?h=rocko\">OpenEmbedded Rocko</a> branch.</field>
</object>
<object model="orm.release" pk="2">
<field type="CharField" name="name">local</field>
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/orm/fixtures/poky.xml b/import-layers/yocto-poky/bitbake/lib/toaster/orm/fixtures/poky.xml
index 7827aac..2f39d77 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/orm/fixtures/poky.xml
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/orm/fixtures/poky.xml
@@ -8,9 +8,9 @@
<!-- Bitbake versions which correspond to the metadata release -->
<object model="orm.bitbakeversion" pk="1">
- <field type="CharField" name="name">pyro</field>
+ <field type="CharField" name="name">rocko</field>
<field type="CharField" name="giturl">git://git.yoctoproject.org/poky</field>
- <field type="CharField" name="branch">pyro</field>
+ <field type="CharField" name="branch">rocko</field>
<field type="CharField" name="dirpath">bitbake</field>
</object>
<object model="orm.bitbakeversion" pk="2">
@@ -29,11 +29,11 @@
<!-- Releases available -->
<object model="orm.release" pk="1">
- <field type="CharField" name="name">pyro</field>
- <field type="CharField" name="description">Yocto Project 2.3 "Pyro"</field>
+ <field type="CharField" name="name">rocko</field>
+ <field type="CharField" name="description">Yocto Project 2.4 "Rocko"</field>
<field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">1</field>
- <field type="CharField" name="branch_name">pyro</field>
- <field type="TextField" name="helptext">Toaster will run your builds using the tip of the <a href="http://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=pyro">Yocto Project Pyro branch</a>.</field>
+ <field type="CharField" name="branch_name">rocko</field>
+ <field type="TextField" name="helptext">Toaster will run your builds using the tip of the <a href="http://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=rocko">Yocto Project Rocko branch</a>.</field>
</object>
<object model="orm.release" pk="2">
<field type="CharField" name="name">local</field>
@@ -105,7 +105,7 @@
<field rel="ManyToOneRel" to="orm.layer" name="layer">1</field>
<field type="IntegerField" name="layer_source">0</field>
<field rel="ManyToOneRel" to="orm.release" name="release">1</field>
- <field type="CharField" name="branch">pyro</field>
+ <field type="CharField" name="branch">rocko</field>
<field type="CharField" name="dirpath">meta</field>
</object>
<object model="orm.layer_version" pk="2">
@@ -136,7 +136,7 @@
<field rel="ManyToOneRel" to="orm.layer" name="layer">2</field>
<field type="IntegerField" name="layer_source">0</field>
<field rel="ManyToOneRel" to="orm.release" name="release">1</field>
- <field type="CharField" name="branch">pyro</field>
+ <field type="CharField" name="branch">rocko</field>
<field type="CharField" name="dirpath">meta-poky</field>
</object>
<object model="orm.layer_version" pk="5">
@@ -167,7 +167,7 @@
<field rel="ManyToOneRel" to="orm.layer" name="layer">3</field>
<field type="IntegerField" name="layer_source">0</field>
<field rel="ManyToOneRel" to="orm.release" name="release">1</field>
- <field type="CharField" name="branch">pyro</field>
+ <field type="CharField" name="branch">rocko</field>
<field type="CharField" name="dirpath">meta-yocto-bsp</field>
</object>
<object model="orm.layer_version" pk="8">
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/orm/management/commands/lsupdates.py b/import-layers/yocto-poky/bitbake/lib/toaster/orm/management/commands/lsupdates.py
index 482908d..efc6b3a 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/orm/management/commands/lsupdates.py
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/orm/management/commands/lsupdates.py
@@ -4,7 +4,7 @@
#
# BitBake Toaster Implementation
#
-# Copyright (C) 2016 Intel Corporation
+# Copyright (C) 2016-2017 Intel Corporation
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@@ -19,10 +19,12 @@
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-from django.core.management.base import NoArgsCommand
+from django.core.management.base import BaseCommand
from orm.models import LayerSource, Layer, Release, Layer_Version
from orm.models import LayerVersionDependency, Machine, Recipe
+from orm.models import Distro
+from orm.models import ToasterSetting
import os
import sys
@@ -56,7 +58,7 @@
self.signal = False
-class Command(NoArgsCommand):
+class Command(BaseCommand):
args = ""
help = "Updates locally cached information from a layerindex server"
@@ -80,6 +82,8 @@
os.system('setterm -cursor off')
self.apiurl = DEFAULT_LAYERINDEX_SERVER
+ if ToasterSetting.objects.filter(name='CUSTOM_LAYERINDEX_SERVER').count() == 1:
+ self.apiurl = ToasterSetting.objects.get(name = 'CUSTOM_LAYERINDEX_SERVER').value
assert self.apiurl is not None
try:
@@ -91,7 +95,9 @@
proxy_settings = os.environ.get("http_proxy", None)
- def _get_json_response(apiurl=DEFAULT_LAYERINDEX_SERVER):
+ def _get_json_response(apiurl=None):
+ if None == apiurl:
+ apiurl=self.apiurl
http_progress = Spinner()
http_progress.start()
@@ -251,6 +257,24 @@
depends_on=lvd)
self.mini_progress("Layer version dependencies", i, total)
+ # update Distros
+ logger.info("Fetching distro information")
+ distros_info = _get_json_response(
+ apilinks['distros'] + "?filter=layerbranch__branch__name:%s" %
+ "OR".join(whitelist_branch_names))
+
+ total = len(distros_info)
+ for i, di in enumerate(distros_info):
+ distro, created = Distro.objects.get_or_create(
+ name=di['name'],
+ layer_version=Layer_Version.objects.get(
+ pk=li_layer_branch_id_to_toaster_lv_id[di['layerbranch']]))
+ distro.up_date = di['updated']
+ distro.name = di['name']
+ distro.description = di['description']
+ distro.save()
+ self.mini_progress("distros", i, total)
+
# update machines
logger.info("Fetching machine information")
machines_info = _get_json_response(
@@ -309,5 +333,5 @@
os.system('setterm -cursor on')
- def handle_noargs(self, **options):
+ def handle(self, **options):
self.update()
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/orm/migrations/0016_clone_progress.py b/import-layers/yocto-poky/bitbake/lib/toaster/orm/migrations/0016_clone_progress.py
new file mode 100644
index 0000000..cd4023b
--- /dev/null
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/orm/migrations/0016_clone_progress.py
@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('orm', '0015_layer_local_source_dir'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='build',
+ name='repos_cloned',
+ field=models.IntegerField(default=1),
+ ),
+ migrations.AddField(
+ model_name='build',
+ name='repos_to_clone',
+ field=models.IntegerField(default=1), # (default off)
+ ),
+ ]
+
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/orm/migrations/0017_distro_clone.py b/import-layers/yocto-poky/bitbake/lib/toaster/orm/migrations/0017_distro_clone.py
new file mode 100644
index 0000000..d3c5901
--- /dev/null
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/orm/migrations/0017_distro_clone.py
@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('orm', '0016_clone_progress'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='Distro',
+ fields=[
+ ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+ ('up_id', models.IntegerField(default=None, null=True)),
+ ('up_date', models.DateTimeField(default=None, null=True)),
+ ('name', models.CharField(max_length=255)),
+ ('description', models.CharField(max_length=255)),
+ ('layer_version', models.ForeignKey(to='orm.Layer_Version')),
+ ],
+ ),
+ ]
+
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/orm/models.py b/import-layers/yocto-poky/bitbake/lib/toaster/orm/models.py
index a49f9a4..3a7dff8 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/orm/models.py
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/orm/models.py
@@ -321,6 +321,22 @@
return queryset
+ def get_available_distros(self):
+ """ Returns QuerySet of all Distros which are provided by the
+ Layers currently added to the Project """
+ queryset = Distro.objects.filter(
+ layer_version__in=self.get_project_layer_versions())
+
+ return queryset
+
+ def get_all_compatible_distros(self):
+ """ Returns QuerySet of all the compatible Wind River distros available to the
+ project including ones from Layers not currently added """
+ queryset = Distro.objects.filter(
+ layer_version__in=self.get_all_compatible_layer_versions())
+
+ return queryset
+
def get_available_recipes(self):
""" Returns QuerySet of all the recipes that are provided by layers
added to this project """
@@ -435,7 +451,13 @@
recipes_to_parse = models.IntegerField(default=1)
# number of recipes parsed so far for this build
- recipes_parsed = models.IntegerField(default=0)
+ recipes_parsed = models.IntegerField(default=1)
+
+ # number of repos to clone for this build
+ repos_to_clone = models.IntegerField(default=1)
+
+ # number of repos cloned so far for this build (default off)
+ repos_cloned = models.IntegerField(default=1)
@staticmethod
def get_recent(project=None):
@@ -486,7 +508,7 @@
tf = Task.objects.filter(build = self)
tfc = tf.count()
if tfc > 0:
- completeper = tf.exclude(order__isnull=True).count()*100 // tfc
+ completeper = tf.exclude(outcome=Task.OUTCOME_NA).count()*100 // tfc
else:
completeper = 0
return completeper
@@ -667,6 +689,13 @@
else:
return False
+ def is_cloning(self):
+ """
+ True if the build is still cloning repos
+ """
+ return self.outcome == Build.IN_PROGRESS and \
+ self.repos_cloned < self.repos_to_clone
+
def is_parsing(self):
"""
True if the build is still parsing recipes
@@ -680,10 +709,11 @@
tasks.
Note that the mechanism for testing whether a Task is "done" is whether
- its order field is set, as per the completeper() method.
+ its outcome field is set, as per the completeper() method.
"""
return self.outcome == Build.IN_PROGRESS and \
- self.task_build.filter(order__isnull=False).count() == 0
+ self.task_build.exclude(outcome=Task.OUTCOME_NA).count() == 0
+
def get_state(self):
"""
@@ -698,6 +728,8 @@
return 'Cancelling';
elif self.is_queued():
return 'Queued'
+ elif self.is_cloning():
+ return 'Cloning'
elif self.is_parsing():
return 'Parsing'
elif self.is_starting():
@@ -1485,12 +1517,12 @@
return self._handle_url_path(self.layer.vcs_web_tree_base_url, '')
def get_vcs_reference(self):
+ if self.commit is not None and len(self.commit) > 0:
+ return self.commit
if self.branch is not None and len(self.branch) > 0:
return self.branch
if self.release is not None:
return self.release.name
- if self.commit is not None and len(self.commit) > 0:
- return self.commit
return 'N/A'
def get_detailspage_url(self, project_id=None):
@@ -1626,7 +1658,7 @@
def get_base_recipe_file(self):
"""Get the base recipe file path if it exists on the file system"""
- path_schema_one = "%s/%s" % (self.base_recipe.layer_version.dirpath,
+ path_schema_one = "%s/%s" % (self.base_recipe.layer_version.local_path,
self.base_recipe.file_path)
path_schema_two = self.base_recipe.file_path
@@ -1780,6 +1812,21 @@
except FileNotFoundError:
logger.info("Stopping existing runbuilds: no current process found")
+class Distro(models.Model):
+ search_allowed_fields = ["name", "description", "layer_version__layer__name"]
+ up_date = models.DateTimeField(null = True, default = None)
+
+ layer_version = models.ForeignKey('Layer_Version')
+ name = models.CharField(max_length=255)
+ description = models.CharField(max_length=255)
+
+ def get_vcs_distro_file_link_url(self):
+ path = self.name+'.conf'
+ return self.layer_version.get_vcs_file_link_url(path)
+
+ def __unicode__(self):
+ return "Distro " + self.name + "(" + self.description + ")"
+
django.db.models.signals.post_save.connect(invalidate_cache)
django.db.models.signals.post_delete.connect(invalidate_cache)
django.db.models.signals.m2m_changed.connect(invalidate_cache)
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/api.py b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/api.py
index 1a6507c..ab6ba69 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/api.py
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/api.py
@@ -18,6 +18,7 @@
# Please run flake8 on this file before sending patches
+import os
import re
import logging
import json
@@ -28,7 +29,7 @@
from orm.models import Recipe, CustomImageRecipe, CustomImagePackage
from orm.models import Layer, Target, Package, Package_Dependency
from orm.models import ProjectVariable
-from bldcontrol.models import BuildRequest
+from bldcontrol.models import BuildRequest, BuildEnvironment
from bldcontrol import bbcontroller
from django.http import HttpResponse, JsonResponse
@@ -509,6 +510,27 @@
(tpackage.package.name, e))
pass
+ # pre-create layer directory structure, so that other builds
+ # are not blocked by this new recipe dependecy
+ # NOTE: this is parallel code to 'localhostbecontroller.py'
+ be = BuildEnvironment.objects.all()[0]
+ layerpath = os.path.join(be.builddir,
+ CustomImageRecipe.LAYER_NAME)
+ for name in ("conf", "recipes"):
+ path = os.path.join(layerpath, name)
+ if not os.path.isdir(path):
+ os.makedirs(path)
+ # pre-create layer.conf
+ config = os.path.join(layerpath, "conf", "layer.conf")
+ if not os.path.isfile(config):
+ with open(config, "w") as conf:
+ conf.write('BBPATH .= ":${LAYERDIR}"\nBBFILES += "${LAYERDIR}/recipes/*.bb"\n')
+ # pre-create new image's recipe file
+ recipe_path = os.path.join(layerpath, "recipes", "%s.bb" %
+ recipe.name)
+ with open(recipe_path, "w") as recipef:
+ recipef.write(recipe.generate_recipe_file_contents())
+
return JsonResponse(
{"error": "ok",
"packages": recipe.get_all_packages().count(),
@@ -752,7 +774,6 @@
return error_response("Package %s not found in excludes"
" but was in included list" %
package.name)
-
else:
recipe.appends_set.add(package)
# Make sure that package is not in the excludes set
@@ -760,26 +781,27 @@
recipe.excludes_set.remove(package)
except:
pass
- # Add the dependencies we think will be added to the recipe
- # as a result of appending this package.
- # TODO this should recurse down the entire deps tree
- for dep in package.package_dependencies_source.all_depends():
- try:
- cust_package = CustomImagePackage.objects.get(
- name=dep.depends_on.name)
- recipe.includes_set.add(cust_package)
- try:
- # When adding the pre-requisite package, make
- # sure it's not in the excluded list from a
- # prior removal.
- recipe.excludes_set.remove(cust_package)
- except package.DoesNotExist:
- # Don't care if the package had never been excluded
- pass
- except:
- logger.warning("Could not add package's suggested"
- "dependencies to the list")
+ # Add the dependencies we think will be added to the recipe
+ # as a result of appending this package.
+ # TODO this should recurse down the entire deps tree
+ for dep in package.package_dependencies_source.all_depends():
+ try:
+ cust_package = CustomImagePackage.objects.get(
+ name=dep.depends_on.name)
+
+ recipe.includes_set.add(cust_package)
+ try:
+ # When adding the pre-requisite package, make
+ # sure it's not in the excluded list from a
+ # prior removal.
+ recipe.excludes_set.remove(cust_package)
+ except package.DoesNotExist:
+ # Don't care if the package had never been excluded
+ pass
+ except:
+ logger.warning("Could not add package's suggested"
+ "dependencies to the list")
return JsonResponse({"error": "ok"})
def delete(self, request, *args, **kwargs):
@@ -797,22 +819,24 @@
recipe.excludes_set.add(package)
else:
recipe.appends_set.remove(package)
- all_current_packages = recipe.get_all_packages()
- reverse_deps_dictlist = self._get_all_dependents(
- package.pk,
- all_current_packages)
+ # remove dependencies as well
+ all_current_packages = recipe.get_all_packages()
- ids = [entry['pk'] for entry in reverse_deps_dictlist]
- reverse_deps = CustomImagePackage.objects.filter(id__in=ids)
- for r in reverse_deps:
- try:
- if r.id in included_packages:
- recipe.excludes_set.add(r)
- else:
- recipe.appends_set.remove(r)
- except:
- pass
+ reverse_deps_dictlist = self._get_all_dependents(
+ package.pk,
+ all_current_packages)
+
+ ids = [entry['pk'] for entry in reverse_deps_dictlist]
+ reverse_deps = CustomImagePackage.objects.filter(id__in=ids)
+ for r in reverse_deps:
+ try:
+ if r.id in included_packages:
+ recipe.excludes_set.add(r)
+ else:
+ recipe.appends_set.remove(r)
+ except:
+ pass
return JsonResponse({"error": "ok"})
except CustomImageRecipe.DoesNotExist:
@@ -874,6 +898,12 @@
machinevar.value = request.POST['machineName']
machinevar.save()
+ # Distro name change
+ if 'distroName' in request.POST:
+ distrovar = prj.projectvariable_set.get(name="DISTRO")
+ distrovar.value = request.POST['distroName']
+ distrovar.save()
+
return JsonResponse({"error": "ok"})
def get(self, request, *args, **kwargs):
@@ -960,10 +990,11 @@
except ProjectVariable.DoesNotExist:
data["machine"] = None
try:
- data["distro"] = project.projectvariable_set.get(
- name="DISTRO").value
+ data["distro"] = {"name":
+ project.projectvariable_set.get(
+ name="DISTRO").value}
except ProjectVariable.DoesNotExist:
- data["distro"] = "-- not set yet"
+ data["distro"] = None
data['error'] = "ok"
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/mrbsection.js b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/mrbsection.js
index 73d0935..c0c5fa9 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/mrbsection.js
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/mrbsection.js
@@ -61,6 +61,12 @@
return (cached.recipes_parsed_percentage !== build.recipes_parsed_percentage);
}
+ // returns true if the number of repos cloned/to clone changed
+ function cloneProgressChanged(build) {
+ var cached = getCached(build);
+ return (cached.repos_cloned_percentage !== build.repos_cloned_percentage);
+ }
+
function refreshMostRecentBuilds(){
libtoaster.getMostRecentBuilds(
libtoaster.ctx.mostRecentBuildsUrl,
@@ -100,6 +106,15 @@
container.html(html);
}
+ else if (cloneProgressChanged(build)) {
+ // update the clone progress text
+ selector = '#repos-cloned-percentage-' + build.id;
+ $(selector).html(build.repos_cloned_percentage);
+
+ // update the recipe progress bar
+ selector = '#repos-cloned-percentage-bar-' + build.id;
+ $(selector).width(build.repos_cloned_percentage + '%');
+ }
else if (tasksProgressChanged(build)) {
// update the task progress text
selector = '#build-pc-done-' + build.id;
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/projectpage.js b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/projectpage.js
index 21adf81..506471e 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/projectpage.js
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/projectpage.js
@@ -15,6 +15,13 @@
var machineInputForm = $("#machine-input-form");
var invalidMachineNameHelp = $("#invalid-machine-name-help");
+ var distroChangeInput = $("#distro-change-input");
+ var distroChangeBtn = $("#distro-change-btn");
+ var distroForm = $("#select-distro-form");
+ var distroChangeFormToggle = $("#change-distro-toggle");
+ var distroNameTitle = $("#project-distro-name");
+ var distroChangeCancel = $("#cancel-distro-change");
+
var freqBuildBtn = $("#freq-build-btn");
var freqBuildList = $("#freq-build-list");
@@ -26,6 +33,7 @@
var currentLayerAddSelection;
var currentMachineAddSelection = "";
+ var currentDistroAddSelection = "";
var urlParams = libtoaster.parseUrlParams();
@@ -45,6 +53,17 @@
updateMachineName(prjInfo.machine.name);
}
+ /* If we're receiving a distro set from the url and it's different from
+ * our current distro then activate set machine sequence.
+ */
+ if (urlParams.hasOwnProperty('setDistro') &&
+ urlParams.setDistro !== prjInfo.distro.name){
+ distroChangeInput.val(urlParams.setDistro);
+ distroChangeBtn.click();
+ } else {
+ updateDistroName(prjInfo.distro.name);
+ }
+
/* Now we're really ready show the page */
$("#project-page").show();
@@ -278,6 +297,60 @@
});
+ /* Change distro functionality */
+
+ distroChangeFormToggle.click(function(){
+ distroForm.slideDown();
+ distroNameTitle.hide();
+ $(this).hide();
+ });
+
+ distroChangeCancel.click(function(){
+ distroForm.slideUp(function(){
+ distroNameTitle.show();
+ distroChangeFormToggle.show();
+ });
+ });
+
+ function updateDistroName(distroName){
+ distroChangeInput.val(distroName);
+ distroNameTitle.text(distroName);
+ }
+
+ libtoaster.makeTypeahead(distroChangeInput,
+ libtoaster.ctx.distrosTypeAheadUrl,
+ { }, function(item){
+ currentDistroAddSelection = item.name;
+ distroChangeBtn.removeAttr("disabled");
+ });
+
+ distroChangeBtn.click(function(e){
+ e.preventDefault();
+ /* We accept any value regardless of typeahead selection or not */
+ if (distroChangeInput.val().length === 0)
+ return;
+
+ currentDistroAddSelection = distroChangeInput.val();
+
+ libtoaster.editCurrentProject(
+ { distroName : currentDistroAddSelection },
+ function(){
+ /* Success machine changed */
+ updateDistroName(currentDistroAddSelection);
+ distroChangeCancel.click();
+
+ /* Show the alert message */
+ var message = $('<span>You have changed the distro to: <strong><span id="notify-machine-name"></span></strong></span>');
+ message.find("#notify-machine-name").text(currentDistroAddSelection);
+ libtoaster.showChangeNotification(message);
+ },
+ function(){
+ /* Failed machine changed */
+ console.warn("Failed to change distro");
+ });
+ });
+
+
/* Change release functionality */
function updateProjectRelease(release){
releaseTitle.text(release.description);
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/projecttopbar.js b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/projecttopbar.js
index 92ab2d6..69220aa 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/projecttopbar.js
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/projecttopbar.js
@@ -73,14 +73,14 @@
newBuildTargetBuildBtn.click(function (e) {
e.preventDefault();
- if (!newBuildTargetInput.val()) {
+ if (!newBuildTargetInput.val().trim()) {
return;
}
/* We use the value of the input field so as to maintain any command also
* added e.g. core-image-minimal:clean and because we can build targets
* that toaster doesn't yet know about
*/
- selectedTarget = { name: newBuildTargetInput.val() };
+ selectedTarget = { name: newBuildTargetInput.val().trim() };
/* Fire off the build */
libtoaster.startABuild(null, selectedTarget.name,
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/tables.py b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/tables.py
index e2d23c1..dca2fa2 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/tables.py
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/tables.py
@@ -23,6 +23,7 @@
from orm.models import Recipe, ProjectLayer, Layer_Version, Machine, Project
from orm.models import CustomImageRecipe, Package, Target, Build, LogMessage, Task
from orm.models import CustomImagePackage, Package_DependencyManager
+from orm.models import Distro
from django.db.models import Q, Max, Sum, Count, When, Case, Value, IntegerField
from django.conf.urls import url
from django.core.urlresolvers import reverse, resolve
@@ -1536,3 +1537,93 @@
context['build_in_progress_none_completed'] = False
return context
+
+
+class DistrosTable(ToasterTable):
+ """Table of Distros in Toaster"""
+
+ def __init__(self, *args, **kwargs):
+ super(DistrosTable, self).__init__(*args, **kwargs)
+ self.empty_state = "Toaster has no distro information for this project. Sadly, distro information cannot be obtained from builds, so this page will remain empty."
+ self.title = "Compatible Distros"
+ self.default_orderby = "name"
+
+ def get_context_data(self, **kwargs):
+ context = super(DistrosTable, self).get_context_data(**kwargs)
+ context['project'] = Project.objects.get(pk=kwargs['pid'])
+ return context
+
+ def setup_filters(self, *args, **kwargs):
+ project = Project.objects.get(pk=kwargs['pid'])
+
+ in_current_project_filter = TableFilter(
+ "in_current_project",
+ "Filter by project Distros"
+ )
+
+ in_project_action = TableFilterActionToggle(
+ "in_project",
+ "Distro provided by layers added to this project",
+ ProjectFilters.in_project(self.project_layers)
+ )
+
+ not_in_project_action = TableFilterActionToggle(
+ "not_in_project",
+ "Distros provided by layers not added to this project",
+ ProjectFilters.not_in_project(self.project_layers)
+ )
+
+ in_current_project_filter.add_action(in_project_action)
+ in_current_project_filter.add_action(not_in_project_action)
+ self.add_filter(in_current_project_filter)
+
+ def setup_queryset(self, *args, **kwargs):
+ prj = Project.objects.get(pk = kwargs['pid'])
+ self.queryset = prj.get_all_compatible_distros()
+ self.queryset = self.queryset.order_by(self.default_orderby)
+
+ self.static_context_extra['current_layers'] = \
+ self.project_layers = \
+ prj.get_project_layer_versions(pk=True)
+
+ def setup_columns(self, *args, **kwargs):
+
+ self.add_column(title="Distro",
+ hideable=False,
+ orderable=True,
+ field_name="name")
+
+ self.add_column(title="Description",
+ field_name="description")
+
+ layer_link_template = '''
+ <a href="{% url 'layerdetails' extra.pid data.layer_version.id %}">
+ {{data.layer_version.layer.name}}</a>
+ '''
+
+ self.add_column(title="Layer",
+ static_data_name="layer_version__layer__name",
+ static_data_template=layer_link_template,
+ orderable=True)
+
+ self.add_column(title="Git revision",
+ help_text="The Git branch, tag or commit. For the layers from the OpenEmbedded layer source, the revision is always the branch compatible with the Yocto Project version you selected for this project",
+ hidden=True,
+ field_name="layer_version__get_vcs_reference")
+
+ wrtemplate_file_template = '''<code>conf/machine/{{data.name}}.conf</code>
+ <a href="{{data.get_vcs_machine_file_link_url}}" target="_blank"><span class="glyphicon glyphicon-new-window"></i></a>'''
+
+ self.add_column(title="Distro file",
+ hidden=True,
+ static_data_name="templatefile",
+ static_data_template=wrtemplate_file_template)
+
+
+ self.add_column(title="Select",
+ help_text="Sets the selected distro to the project",
+ hideable=False,
+ filter_name="in_current_project",
+ static_data_name="add-del-layers",
+ static_data_template='{% include "distro_btn.html" %}')
+
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/base.html b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/base.html
index 32b4979..4f72064 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/base.html
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/base.html
@@ -49,6 +49,7 @@
recipesTypeAheadUrl: {% url 'xhr_recipestypeahead' project.id as paturl%}{{paturl|json}},
layersTypeAheadUrl: {% url 'xhr_layerstypeahead' project.id as paturl%}{{paturl|json}},
machinesTypeAheadUrl: {% url 'xhr_machinestypeahead' project.id as paturl%}{{paturl|json}},
+ distrosTypeAheadUrl: {% url 'xhr_distrostypeahead' project.id as paturl%}{{paturl|json}},
projectBuildsUrl: {% url 'projectbuilds' project.id as pburl %}{{pburl|json}},
xhrCustomRecipeUrl : "{% url 'xhr_customrecipe' %}",
projectId : {{project.id}},
@@ -109,6 +110,7 @@
All builds
</a>
</li>
+ {% if project_enable %}
<li id="navbar-all-projects"
{% if request.resolver_match.url_name == 'all-projects' %}
class="active"
@@ -118,6 +120,7 @@
All projects
</a>
</li>
+ {% endif %}
{% endif %}
<li id="navbar-docs">
<a target="_blank" href="http://www.yoctoproject.org/docs/latest/toaster-manual/toaster-manual.html">
@@ -126,7 +129,9 @@
</a>
</li>
</ul>
+ {% if project_enable %}
<a class="btn btn-default navbar-btn navbar-right" id="new-project-button" href="{% url 'newproject' %}">New project</a>
+ {% endif %}
</div>
</div>
</nav>
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/baseprojectpage.html b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/baseprojectpage.html
index 8427d25..f2bb2eb 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/baseprojectpage.html
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/baseprojectpage.html
@@ -32,6 +32,7 @@
<li><a href="{% url 'projectsoftwarerecipes' project.id %}">Software recipes</a></li>
<li><a href="{% url 'projectmachines' project.id %}">Machines</a></li>
<li><a href="{% url 'projectlayers' project.id %}">Layers</a></li>
+ <li><a href="{% url 'projectdistros' project.id %}">Distros</a></li>
<li class="nav-header">Extra configuration</li>
<li><a href="{% url 'projectconf' project.id %}">BitBake variables</a></li>
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/distro_btn.html b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/distro_btn.html
new file mode 100644
index 0000000..fac7947
--- /dev/null
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/distro_btn.html
@@ -0,0 +1,20 @@
+<a href="{% url 'project' extra.pid %}?setDistro={{data.name}}" class="btn btn-default btn-block layer-exists-{{data.layer_version.id}}"
+ {% if data.layer_version.pk not in extra.current_layers %}
+ style="display:none;"
+ {% endif %}>
+ Set distro</a>
+<a class="btn btn-default btn-block layerbtn layer-add-{{data.layer_version.id}}" data-layer='{
+ "id": {{data.layer_version.id}},
+ "name": "{{data.layer_version.layer.name}}",
+ "xhrLayerUrl": "{% url "xhr_layer" extra.pid data.pk %}",
+ "layerdetailurl": "{%url 'layerdetails' extra.pid data.layer_version.id %}"
+ }' data-directive="add"
+ {% if data.layer_version.pk in extra.current_layers %}
+ style="display:none;"
+ {% endif %}
+>
+ <span class="glyphicon glyphicon-plus"></span>
+ Add layer
+ <span class="glyphicon glyphicon-question-sign get-help" title="To select this distro, you must first add the {{data.layer_version.layer.name}} layer to your project"></i>
+</a>
+
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/health.html b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/health.html
new file mode 100644
index 0000000..f17fdbc
--- /dev/null
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/health.html
@@ -0,0 +1,6 @@
+<!DOCTYPE html>
+<html lang="en">
+ <head><title>Toaster Health</title></head>
+ <body>Ok</body>
+</html>
+
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/landing.html b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/landing.html
index 4986632..70c7359 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/landing.html
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/landing.html
@@ -14,12 +14,20 @@
<p>A web interface to <a href="http://www.openembedded.org">OpenEmbedded</a> and <a href="http://www.yoctoproject.org/tools-resources/projects/bitbake">BitBake</a>, the <a href="http://www.yoctoproject.org">Yocto Project</a> build system.</p>
+ <p class="top-air">
+ <a class="btn btn-info btn-lg" href="http://www.yoctoproject.org/docs/latest/toaster-manual/toaster-manual.html#toaster-manual-setup-and-use">
+ Toaster is ready to capture your command line builds
+ </a>
+ </p>
+
{% if lvs_nos %}
+ {% if project_enable %}
<p class="top-air">
<a class="btn btn-primary btn-lg" href="{% url 'newproject' %}">
- To start building, create your first Toaster project
+ Create your first Toaster project to run manage builds
</a>
</p>
+ {% endif %}
{% else %}
<div class="alert alert-info lead top-air">
Toaster has no layer information. Without layer information, you cannot run builds. To generate layer information you can:
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/mrb_section.html b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/mrb_section.html
index b761ffe..c5b9fe9 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/mrb_section.html
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/mrb_section.html
@@ -64,7 +64,9 @@
</div>
<div data-build-state="<%:state%>">
- <%if state == 'Parsing'%>
+ <%if state == 'Cloning'%>
+ <%include tmpl='#cloning-repos-build-template'/%>
+ <%else state == 'Parsing'%>
<%include tmpl='#parsing-recipes-build-template'/%>
<%else state == 'Queued'%>
<%include tmpl='#queued-build-template'/%>
@@ -98,6 +100,31 @@
</div>
</script>
+<!-- cloning repos build -->
+<script id="cloning-repos-build-template" type="text/x-jsrender">
+ <!-- progress bar and parse completion percentage -->
+ <div data-role="build-status" class="col-md-4 col-md-offset-1 progress-info">
+ <!-- progress bar -->
+ <div class="progress">
+ <div id="repos-cloned-percentage-bar-<%:id%>"
+ style="width: <%:repos_cloned_percentage%>%;"
+ class="progress-bar">
+ </div>
+ </div>
+ </div>
+
+ <div class="col-md-4 progress-info">
+ <!-- parse completion percentage -->
+ <span class="glyphicon glyphicon-question-sign get-help get-help-blue"
+ title="Toaster is cloning the repos required for your build">
+ </span>
+
+ Cloning <span id="repos-cloned-percentage-<%:id%>"><%:repos_cloned_percentage%></span>% complete
+
+ <%include tmpl='#cancel-template'/%>
+ </div>
+</script>
+
<!-- parsing recipes build -->
<script id="parsing-recipes-build-template" type="text/x-jsrender">
<!-- progress bar and parse completion percentage -->
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/project.html b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/project.html
index ab7e665..11603d1 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/project.html
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/project.html
@@ -77,6 +77,22 @@
</form>
</div>
+ <div class="well well-transparent" id="distro-section">
+ <h3>Distro</h3>
+
+ <p class="lead"><span id="project-distro-name"></span> <span class="glyphicon glyphicon-edit" id="change-distro-toggle"></span></p>
+
+ <form id="select-distro-form" style="display:none;" class="form-inline">
+ <span class="help-block">Distro suggestions come from the Layer Index</a></span>
+ <div class="form-group">
+ <input class="form-control" id="distro-change-input" autocomplete="off" value="" data-provide="typeahead" data-minlength="1" data-autocomplete="off" type="text">
+ </div>
+ <button id="distro-change-btn" class="btn btn-default" type="button">Save</button>
+ <a href="#" id="cancel-distro-change" class="btn btn-link">Cancel</a>
+ <p class="form-link"><a href="{% url 'projectdistros' project.id %}">View compatible distros</a></p>
+ </form>
+ </div>
+
<div class="well well-transparent">
<h3>Most built recipes</h3>
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/typeaheads.py b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/typeaheads.py
index 58c650f..5aa0f8d 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/typeaheads.py
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/typeaheads.py
@@ -100,6 +100,36 @@
return results
+class DistrosTypeAhead(ToasterTypeAhead):
+ """ Typeahead for all the distros available in the current project's
+ configuration """
+ def __init__(self):
+ super(DistrosTypeAhead, self).__init__()
+
+ def apply_search(self, search_term, prj, request):
+ distros = prj.get_available_distros()
+ distros = distros.order_by("name")
+
+ primary_results = distros.filter(name__istartswith=search_term)
+ secondary_results = distros.filter(name__icontains=search_term).exclude(pk__in=primary_results)
+ tertiary_results = distros.filter(layer_version__layer__name__icontains=search_term).exclude(pk__in=primary_results).exclude(pk__in=secondary_results)
+
+ results = []
+
+ for distro in list(primary_results) + list(secondary_results) + list(tertiary_results):
+
+ detail = "[ %s ]" % (distro.layer_version.layer.name)
+ needed_fields = {
+ 'id' : distro.pk,
+ 'name' : distro.name,
+ 'detail' : detail,
+ }
+
+ results.append(needed_fields)
+
+ return results
+
+
class RecipesTypeAhead(ToasterTypeAhead):
""" Typeahead for all the recipes available in the current project's
configuration """
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/urls.py b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/urls.py
index d92f190..e07b0ef 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/urls.py
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/urls.py
@@ -1,7 +1,7 @@
#
# BitBake Toaster Implementation
#
-# Copyright (C) 2013 Intel Corporation
+# Copyright (C) 2013-2017 Intel Corporation
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@@ -16,7 +16,7 @@
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-from django.conf.urls import patterns, include, url
+from django.conf.urls import include, url
from django.views.generic import RedirectView, TemplateView
from django.http import HttpResponseBadRequest
@@ -25,49 +25,50 @@
from toastergui import typeaheads
from toastergui import api
from toastergui import widgets
+from toastergui import views
-urlpatterns = patterns('toastergui.views',
+urlpatterns = [
# landing page
- url(r'^landing/$', 'landing', name='landing'),
+ url(r'^landing/$', views.landing, name='landing'),
url(r'^builds/$',
tables.AllBuildsTable.as_view(template_name="builds-toastertable.html"),
name='all-builds'),
# build info navigation
- url(r'^build/(?P<build_id>\d+)$', 'builddashboard', name="builddashboard"),
+ url(r'^build/(?P<build_id>\d+)$', views.builddashboard, name="builddashboard"),
url(r'^build/(?P<build_id>\d+)/tasks/$',
buildtables.BuildTasksTable.as_view(
template_name="buildinfo-toastertable.html"),
name='tasks'),
- url(r'^build/(?P<build_id>\d+)/task/(?P<task_id>\d+)$', 'task', name='task'),
+ url(r'^build/(?P<build_id>\d+)/task/(?P<task_id>\d+)$', views.task, name='task'),
url(r'^build/(?P<build_id>\d+)/recipes/$',
buildtables.BuiltRecipesTable.as_view(
template_name="buildinfo-toastertable.html"),
name='recipes'),
- url(r'^build/(?P<build_id>\d+)/recipe/(?P<recipe_id>\d+)/active_tab/(?P<active_tab>\d{1})$', 'recipe', name='recipe'),
+ url(r'^build/(?P<build_id>\d+)/recipe/(?P<recipe_id>\d+)/active_tab/(?P<active_tab>\d{1})$', views.recipe, name='recipe'),
- url(r'^build/(?P<build_id>\d+)/recipe/(?P<recipe_id>\d+)$', 'recipe', name='recipe'),
- url(r'^build/(?P<build_id>\d+)/recipe_packages/(?P<recipe_id>\d+)$', 'recipe_packages', name='recipe_packages'),
+ url(r'^build/(?P<build_id>\d+)/recipe/(?P<recipe_id>\d+)$', views.recipe, name='recipe'),
+ url(r'^build/(?P<build_id>\d+)/recipe_packages/(?P<recipe_id>\d+)$', views.recipe_packages, name='recipe_packages'),
url(r'^build/(?P<build_id>\d+)/packages/$',
buildtables.BuiltPackagesTable.as_view(
template_name="buildinfo-toastertable.html"),
name='packages'),
- url(r'^build/(?P<build_id>\d+)/package/(?P<package_id>\d+)$', 'package_built_detail',
+ url(r'^build/(?P<build_id>\d+)/package/(?P<package_id>\d+)$', views.package_built_detail,
name='package_built_detail'),
url(r'^build/(?P<build_id>\d+)/package_built_dependencies/(?P<package_id>\d+)$',
- 'package_built_dependencies', name='package_built_dependencies'),
+ views.package_built_dependencies, name='package_built_dependencies'),
url(r'^build/(?P<build_id>\d+)/package_included_detail/(?P<target_id>\d+)/(?P<package_id>\d+)$',
- 'package_included_detail', name='package_included_detail'),
+ views.package_included_detail, name='package_included_detail'),
url(r'^build/(?P<build_id>\d+)/package_included_dependencies/(?P<target_id>\d+)/(?P<package_id>\d+)$',
- 'package_included_dependencies', name='package_included_dependencies'),
+ views.package_included_dependencies, name='package_included_dependencies'),
url(r'^build/(?P<build_id>\d+)/package_included_reverse_dependencies/(?P<target_id>\d+)/(?P<package_id>\d+)$',
- 'package_included_reverse_dependencies', name='package_included_reverse_dependencies'),
+ views.package_included_reverse_dependencies, name='package_included_reverse_dependencies'),
url(r'^build/(?P<build_id>\d+)/target/(?P<target_id>\d+)$',
buildtables.InstalledPackagesTable.as_view(
@@ -75,11 +76,11 @@
name='target'),
- url(r'^dentries/build/(?P<build_id>\d+)/target/(?P<target_id>\d+)$', 'xhr_dirinfo', name='dirinfo_ajax'),
- url(r'^build/(?P<build_id>\d+)/target/(?P<target_id>\d+)/dirinfo$', 'dirinfo', name='dirinfo'),
- url(r'^build/(?P<build_id>\d+)/target/(?P<target_id>\d+)/dirinfo_filepath/_(?P<file_path>(?:/[^/\n]+)*)$', 'dirinfo', name='dirinfo_filepath'),
- url(r'^build/(?P<build_id>\d+)/configuration$', 'configuration', name='configuration'),
- url(r'^build/(?P<build_id>\d+)/configvars$', 'configvars', name='configvars'),
+ url(r'^dentries/build/(?P<build_id>\d+)/target/(?P<target_id>\d+)$', views.xhr_dirinfo, name='dirinfo_ajax'),
+ url(r'^build/(?P<build_id>\d+)/target/(?P<target_id>\d+)/dirinfo$', views.dirinfo, name='dirinfo'),
+ url(r'^build/(?P<build_id>\d+)/target/(?P<target_id>\d+)/dirinfo_filepath/_(?P<file_path>(?:/[^/\n]+)*)$', views.dirinfo, name='dirinfo_filepath'),
+ url(r'^build/(?P<build_id>\d+)/configuration$', views.configuration, name='configuration'),
+ url(r'^build/(?P<build_id>\d+)/configvars$', views.configvars, name='configvars'),
url(r'^build/(?P<build_id>\d+)/buildtime$',
buildtables.BuildTimeTable.as_view(
template_name="buildinfo-toastertable.html"),
@@ -97,26 +98,26 @@
# image information dir
url(r'^build/(?P<build_id>\d+)/target/(?P<target_id>\d+)/packagefile/(?P<packagefile_id>\d+)$',
- 'image_information_dir', name='image_information_dir'),
+ views.image_information_dir, name='image_information_dir'),
# build download artifact
- url(r'^build/(?P<build_id>\d+)/artifact/(?P<artifact_type>\w+)/id/(?P<artifact_id>\w+)', 'build_artifact', name="build_artifact"),
+ url(r'^build/(?P<build_id>\d+)/artifact/(?P<artifact_type>\w+)/id/(?P<artifact_id>\w+)', views.build_artifact, name="build_artifact"),
# project URLs
- url(r'^newproject/$', 'newproject', name='newproject'),
+ url(r'^newproject/$', views.newproject, name='newproject'),
url(r'^projects/$',
tables.ProjectsTable.as_view(template_name="projects-toastertable.html"),
name='all-projects'),
- url(r'^project/(?P<pid>\d+)/$', 'project', name='project'),
- url(r'^project/(?P<pid>\d+)/configuration$', 'projectconf', name='projectconf'),
+ url(r'^project/(?P<pid>\d+)/$', views.project, name='project'),
+ url(r'^project/(?P<pid>\d+)/configuration$', views.projectconf, name='projectconf'),
url(r'^project/(?P<pid>\d+)/builds/$',
tables.ProjectBuildsTable.as_view(template_name="projectbuilds-toastertable.html"),
name='projectbuilds'),
# the import layer is a project-specific functionality;
- url(r'^project/(?P<pid>\d+)/importlayer$', 'importlayer', name='importlayer'),
+ url(r'^project/(?P<pid>\d+)/importlayer$', views.importlayer, name='importlayer'),
# the table pages that have been converted to ToasterTable widget
url(r'^project/(?P<pid>\d+)/machines/$',
@@ -142,7 +143,7 @@
name="projectlayers"),
url(r'^project/(?P<pid>\d+)/layer/(?P<layerid>\d+)$',
- 'layerdetails', name='layerdetails'),
+ views.layerdetails, name='layerdetails'),
url(r'^project/(?P<pid>\d+)/layer/(?P<layerid>\d+)/recipes/$',
tables.LayerRecipesTable.as_view(template_name="generic-toastertable-page.html"),
@@ -157,6 +158,11 @@
name=tables.LayerMachinesTable.__name__.lower()),
+ url(r'^project/(?P<pid>\d+)/distros/$',
+ tables.DistrosTable.as_view(template_name="generic-toastertable-page.html"),
+ name="projectdistros"),
+
+
url(r'^project/(?P<pid>\d+)/customrecipe/(?P<custrecipeid>\d+)/selectpackages/$',
tables.SelectPackagesTable.as_view(), name="recipeselectpackages"),
@@ -166,7 +172,7 @@
name="customrecipe"),
url(r'^project/(?P<pid>\d+)/customrecipe/(?P<recipe_id>\d+)/download$',
- 'customrecipe_download',
+ views.customrecipe_download,
name="customrecipedownload"),
url(r'^project/(?P<pid>\d+)/recipe/(?P<recipe_id>\d+)$',
@@ -186,9 +192,12 @@
typeaheads.GitRevisionTypeAhead.as_view(),
name='xhr_gitrevtypeahead'),
- url(r'^xhr_testreleasechange/(?P<pid>\d+)$', 'xhr_testreleasechange',
+ url(r'^xhr_typeahead/(?P<pid>\d+)/distros$',
+ typeaheads.DistrosTypeAhead.as_view(), name='xhr_distrostypeahead'),
+
+ url(r'^xhr_testreleasechange/(?P<pid>\d+)$', views.xhr_testreleasechange,
name='xhr_testreleasechange'),
- url(r'^xhr_configvaredit/(?P<pid>\d+)$', 'xhr_configvaredit',
+ url(r'^xhr_configvaredit/(?P<pid>\d+)$', views.xhr_configvaredit,
name='xhr_configvaredit'),
url(r'^xhr_layer/(?P<pid>\d+)/(?P<layerversion_id>\d+)$',
@@ -200,7 +209,7 @@
name='xhr_layer'),
# JS Unit tests
- url(r'^js-unit-tests/$', 'jsunittests', name='js-unit-tests'),
+ url(r'^js-unit-tests/$', views.jsunittests, name='js-unit-tests'),
# image customisation functionality
url(r'^xhr_customrecipe/(?P<recipe_id>\d+)'
@@ -235,6 +244,11 @@
url(r'^mostrecentbuilds$', widgets.MostRecentBuildsView.as_view(),
name='most_recent_builds'),
- # default redirection
+ # JSON data for aggregators
+ url(r'^api/builds$', views.json_builds, name='json_builds'),
+ url(r'^api/building$', views.json_building, name='json_building'),
+ url(r'^api/build/(?P<build_id>\d+)$', views.json_build, name='json_build'),
+
+ # default redirection
url(r'^$', RedirectView.as_view(url='landing', permanent=True)),
-)
+]
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/views.py b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/views.py
index 75c5911..34ed2b2 100755
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/views.py
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/views.py
@@ -35,7 +35,7 @@
from django.core.urlresolvers import reverse, resolve
from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
-from django.http import HttpResponseNotFound
+from django.http import HttpResponseNotFound, JsonResponse
from django.utils import timezone
from datetime import timedelta, datetime
from toastergui.templatetags.projecttags import json as jsonfilter
@@ -49,6 +49,8 @@
logger = logging.getLogger("toaster")
+# Project creation and managed build enable
+project_enable = ('1' == os.environ.get('TOASTER_BUILDSERVER'))
class MimeTypeFinder(object):
# setting this to False enables additional non-standard mimetypes
@@ -65,6 +67,12 @@
guessed_type = 'application/octet-stream'
return guessed_type
+# single point to add global values into the context before rendering
+def toaster_render(request, page, context):
+ context['project_enable'] = project_enable
+ return render(request, page, context)
+
+
# all new sessions should come through the landing page;
# determine in which mode we are running in, and redirect appropriately
def landing(request):
@@ -86,7 +94,7 @@
context = {'lvs_nos' : Layer_Version.objects.all().count()}
- return render(request, 'landing.html', context)
+ return toaster_render(request, 'landing.html', context)
def objtojson(obj):
from django.db.models.query import QuerySet
@@ -277,7 +285,7 @@
return None, invalid + str(field_input_list)
# Check we are looking for a valid field
- valid_fields = model._meta.get_all_field_names()
+ valid_fields = [f.name for f in model._meta.get_fields()]
for field in field_input_list[0].split(AND_VALUE_SEPARATOR):
if True in [field.startswith(x) for x in valid_fields]:
break
@@ -457,10 +465,15 @@
npkg = 0
pkgsz = 0
package = None
- for package in Package.objects.filter(id__in = [x.package_id for x in t.target_installed_package_set.all()]):
- pkgsz = pkgsz + package.size
- if package.installed_name:
- npkg = npkg + 1
+ # Chunk the query to avoid "too many SQL variables" error
+ package_set = t.target_installed_package_set.all()
+ package_set_len = len(package_set)
+ for ps_start in range(0,package_set_len,500):
+ ps_stop = min(ps_start+500,package_set_len)
+ for package in Package.objects.filter(id__in = [x.package_id for x in package_set[ps_start:ps_stop]]):
+ pkgsz = pkgsz + package.size
+ if package.installed_name:
+ npkg = npkg + 1
elem['npkg'] = npkg
elem['pkgsz'] = pkgsz
ti = Target_Image_File.objects.filter(target_id = t.id)
@@ -514,7 +527,7 @@
'packagecount' : packageCount,
'logmessages' : logmessages,
}
- return render( request, template, context )
+ return toaster_render( request, template, context )
@@ -586,7 +599,7 @@
build__completed_on__lt=task_object.build.completed_on).exclude(
order__isnull=True).exclude(outcome=Task.OUTCOME_NA).order_by('-build__completed_on')
- return render( request, template, context )
+ return toaster_render( request, template, context )
def recipe(request, build_id, recipe_id, active_tab="1"):
template = "recipe.html"
@@ -613,7 +626,7 @@
'package_count' : package_count,
'tab_states' : tab_states,
}
- return render(request, template, context)
+ return toaster_render(request, template, context)
def recipe_packages(request, build_id, recipe_id):
template = "recipe_packages.html"
@@ -658,7 +671,7 @@
},
]
}
- response = render(request, template, context)
+ response = toaster_render(request, template, context)
_set_parameters_values(pagesize, orderby, request)
return response
@@ -780,7 +793,7 @@
'dir_list': dir_list,
'file_path': file_path,
}
- return render(request, template, context)
+ return toaster_render(request, template, context)
def _find_task_dep(task_object):
tdeps = Task_Dependency.objects.filter(task=task_object).filter(depends_on__order__gt=0)
@@ -832,7 +845,7 @@
'build': build,
'project': build.project,
'targets': Target.objects.filter(build=build_id)})
- return render(request, template, context)
+ return toaster_render(request, template, context)
def configvars(request, build_id):
@@ -921,7 +934,7 @@
],
}
- response = render(request, template, context)
+ response = toaster_render(request, template, context)
_set_parameters_values(pagesize, orderby, request)
return response
@@ -934,7 +947,7 @@
'project': build.project,
'objects' : files
}
- return render(request, template, context)
+ return toaster_render(request, template, context)
# A set of dependency types valid for both included and built package views
@@ -1087,7 +1100,7 @@
if paths.all().count() < 2:
context['disable_sort'] = True;
- response = render(request, template, context)
+ response = toaster_render(request, template, context)
_set_parameters_values(pagesize, orderby, request)
return response
@@ -1106,7 +1119,7 @@
'other_deps' : dependencies['other_deps'],
'dependency_count' : _get_package_dependency_count(package, -1, False)
}
- return render(request, template, context)
+ return toaster_render(request, template, context)
def package_included_detail(request, build_id, target_id, package_id):
@@ -1152,7 +1165,7 @@
}
if paths.all().count() < 2:
context['disable_sort'] = True
- response = render(request, template, context)
+ response = toaster_render(request, template, context)
_set_parameters_values(pagesize, orderby, request)
return response
@@ -1176,7 +1189,7 @@
'reverse_count' : _get_package_reverse_dep_count(package, target_id),
'dependency_count' : _get_package_dependency_count(package, target_id, True)
}
- return render(request, template, context)
+ return toaster_render(request, template, context)
def package_included_reverse_dependencies(request, build_id, target_id, package_id):
template = "package_included_reverse_dependencies.html"
@@ -1227,7 +1240,7 @@
}
if objects.all().count() < 2:
context['disable_sort'] = True
- response = render(request, template, context)
+ response = toaster_render(request, template, context)
_set_parameters_values(pagesize, orderby, request)
return response
@@ -1251,6 +1264,89 @@
}
return ret
+# REST-based API calls to return build/building status to external Toaster
+# managers and aggregators via JSON
+
+def _json_build_status(build_id,extend):
+ build_stat = None
+ try:
+ build = Build.objects.get( pk = build_id )
+ build_stat = {}
+ build_stat['id'] = build.id
+ build_stat['name'] = build.build_name
+ build_stat['machine'] = build.machine
+ build_stat['distro'] = build.distro
+ build_stat['start'] = build.started_on
+ # look up target name
+ target= Target.objects.get( build = build )
+ if target:
+ if target.task:
+ build_stat['target'] = '%s:%s' % (target.target,target.task)
+ else:
+ build_stat['target'] = '%s' % (target.target)
+ else:
+ build_stat['target'] = ''
+ # look up project name
+ project = Project.objects.get( build = build )
+ if project:
+ build_stat['project'] = project.name
+ else:
+ build_stat['project'] = ''
+ if Build.IN_PROGRESS == build.outcome:
+ now = timezone.now()
+ timediff = now - build.started_on
+ build_stat['seconds']='%.3f' % timediff.total_seconds()
+ build_stat['clone']='%d:%d' % (build.repos_cloned,build.repos_to_clone)
+ build_stat['parse']='%d:%d' % (build.recipes_parsed,build.recipes_to_parse)
+ tf = Task.objects.filter(build = build)
+ tfc = tf.count()
+ if tfc > 0:
+ tfd = tf.exclude(order__isnull=True).count()
+ else:
+ tfd = 0
+ build_stat['task']='%d:%d' % (tfd,tfc)
+ else:
+ build_stat['outcome'] = build.get_outcome_text()
+ timediff = build.completed_on - build.started_on
+ build_stat['seconds']='%.3f' % timediff.total_seconds()
+ build_stat['stop'] = build.completed_on
+ messages = LogMessage.objects.all().filter(build = build)
+ errors = len(messages.filter(level=LogMessage.ERROR) |
+ messages.filter(level=LogMessage.EXCEPTION) |
+ messages.filter(level=LogMessage.CRITICAL))
+ build_stat['errors'] = errors
+ warnings = len(messages.filter(level=LogMessage.WARNING))
+ build_stat['warnings'] = warnings
+ if extend:
+ build_stat['cooker_log'] = build.cooker_log_path
+ except Exception as e:
+ build_state = str(e)
+ return build_stat
+
+def json_builds(request):
+ build_table = []
+ builds = []
+ try:
+ builds = Build.objects.exclude(outcome=Build.IN_PROGRESS).order_by("-started_on")
+ for build in builds:
+ build_table.append(_json_build_status(build.id,False))
+ except Exception as e:
+ build_table = str(e)
+ return JsonResponse({'builds' : build_table, 'count' : len(builds)})
+
+def json_building(request):
+ build_table = []
+ builds = []
+ try:
+ builds = Build.objects.filter(outcome=Build.IN_PROGRESS).order_by("-started_on")
+ for build in builds:
+ build_table.append(_json_build_status(build.id,False))
+ except Exception as e:
+ build_table = str(e)
+ return JsonResponse({'building' : build_table, 'count' : len(builds)})
+
+def json_build(request,build_id):
+ return JsonResponse({'build' : _json_build_status(build_id,True)})
import toastermain.settings
@@ -1277,6 +1373,9 @@
# new project
def newproject(request):
+ if not project_enable:
+ return redirect( landing )
+
template = "newproject.html"
context = {
'email': request.user.email if request.user.is_authenticated() else '',
@@ -1291,7 +1390,7 @@
if request.method == "GET":
# render new project page
- return render(request, template, context)
+ return toaster_render(request, template, context)
elif request.method == "POST":
mandatory_fields = ['projectname', 'ptype']
try:
@@ -1331,7 +1430,7 @@
context['alert'] = "Your chosen username is already used"
else:
context['alert'] = str(e)
- return render(request, template, context)
+ return toaster_render(request, template, context)
raise Exception("Invalid HTTP method for this page")
@@ -1339,7 +1438,7 @@
def project(request, pid):
project = Project.objects.get(pk=pid)
context = {"project": project}
- return render(request, "project.html", context)
+ return toaster_render(request, "project.html", context)
def jsunittests(request):
""" Provides a page for the js unit tests """
@@ -1365,7 +1464,7 @@
name="MACHINE",
value="qemux86")
context = {'project': new_project}
- return render(request, "js-unit-tests.html", context)
+ return toaster_render(request, "js-unit-tests.html", context)
from django.views.decorators.csrf import csrf_exempt
@csrf_exempt
@@ -1500,7 +1599,7 @@
context = {
'project': Project.objects.get(id=pid),
}
- return render(request, template, context)
+ return toaster_render(request, template, context)
def layerdetails(request, pid, layerid):
project = Project.objects.get(pk=pid)
@@ -1529,7 +1628,7 @@
'projectlayers': list(project_layers)
}
- return render(request, 'layerdetails.html', context)
+ return toaster_render(request, 'layerdetails.html', context)
def get_project_configvars_context():
@@ -1619,7 +1718,7 @@
except (ProjectVariable.DoesNotExist, BuildEnvironment.DoesNotExist):
pass
- return render(request, "projectconf.html", context)
+ return toaster_render(request, "projectconf.html", context)
def _file_names_for_artifact(build, artifact_type, artifact_id):
"""
@@ -1686,6 +1785,7 @@
return response
else:
- return render(request, "unavailable_artifact.html")
+ return toaster_render(request, "unavailable_artifact.html")
except (ObjectDoesNotExist, IOError):
- return render(request, "unavailable_artifact.html")
+ return toaster_render(request, "unavailable_artifact.html")
+
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/widgets.py b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/widgets.py
index 6b7b981..a1792d9 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/widgets.py
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/widgets.py
@@ -41,6 +41,7 @@
import json
import collections
import re
+import os
from toastergui.tablefilter import TableFilterMap
@@ -86,6 +87,9 @@
context['table_name'] = type(self).__name__.lower()
context['empty_state'] = self.empty_state
+ # global variables
+ context['project_enable'] = ('1' == os.environ.get('TOASTER_BUILDSERVER'))
+
return context
def get(self, request, *args, **kwargs):
@@ -511,6 +515,10 @@
int((build_obj.recipes_parsed /
build_obj.recipes_to_parse) * 100)
+ build['repos_cloned_percentage'] = \
+ int((build_obj.repos_cloned /
+ build_obj.repos_to_clone) * 100)
+
tasks_complete_percentage = 0
if build_obj.outcome in (Build.SUCCEEDED, Build.FAILED):
tasks_complete_percentage = 100
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastermain/management/commands/buildslist.py b/import-layers/yocto-poky/bitbake/lib/toaster/toastermain/management/commands/buildslist.py
index 8dfef0a..70b5812 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastermain/management/commands/buildslist.py
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastermain/management/commands/buildslist.py
@@ -1,13 +1,13 @@
-from django.core.management.base import NoArgsCommand, CommandError
+from django.core.management.base import BaseCommand, CommandError
from orm.models import Build
import os
-class Command(NoArgsCommand):
+class Command(BaseCommand):
args = ""
help = "Lists current builds"
- def handle_noargs(self,**options):
+ def handle(self,**options):
for b in Build.objects.all():
print("%d: %s %s %s" % (b.pk, b.machine, b.distro, ",".join([x.target for x in b.target_set.all()])))
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastermain/settings.py b/import-layers/yocto-poky/bitbake/lib/toaster/toastermain/settings.py
index 1fd649c..13541d3 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastermain/settings.py
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastermain/settings.py
@@ -24,7 +24,6 @@
import os
DEBUG = True
-TEMPLATE_DEBUG = DEBUG
# Set to True to see the SQL queries in console
SQL_DEBUG = False
@@ -161,12 +160,47 @@
# Make this unique, and don't share it with anybody.
SECRET_KEY = 'NOT_SUITABLE_FOR_HOSTED_DEPLOYMENT'
-# List of callables that know how to import templates from various sources.
-TEMPLATE_LOADERS = (
- 'django.template.loaders.filesystem.Loader',
- 'django.template.loaders.app_directories.Loader',
-# 'django.template.loaders.eggs.Loader',
-)
+class InvalidString(str):
+ def __mod__(self, other):
+ from django.template.base import TemplateSyntaxError
+ raise TemplateSyntaxError(
+ "Undefined variable or unknown value for: \"%s\"" % other)
+
+TEMPLATES = [
+ {
+ 'BACKEND': 'django.template.backends.django.DjangoTemplates',
+ 'DIRS': [
+ # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
+ # Always use forward slashes, even on Windows.
+ # Don't forget to use absolute paths, not relative paths.
+ ],
+ 'OPTIONS': {
+ 'context_processors': [
+ # Insert your TEMPLATE_CONTEXT_PROCESSORS here or use this
+ # list if you haven't customized them:
+ 'django.contrib.auth.context_processors.auth',
+ 'django.template.context_processors.debug',
+ 'django.template.context_processors.i18n',
+ 'django.template.context_processors.media',
+ 'django.template.context_processors.static',
+ 'django.template.context_processors.tz',
+ 'django.contrib.messages.context_processors.messages',
+ # Custom
+ 'django.template.context_processors.request',
+ 'toastergui.views.managedcontextprocessor',
+
+ ],
+ 'loaders': [
+ # List of callables that know how to import templates from various sources.
+ 'django.template.loaders.filesystem.Loader',
+ 'django.template.loaders.app_directories.Loader',
+ #'django.template.loaders.eggs.Loader',
+ ],
+ 'string_if_invalid': InvalidString("%s"),
+ 'debug': DEBUG,
+ },
+ },
+]
MIDDLEWARE_CLASSES = (
'django.middleware.common.CommonMiddleware',
@@ -203,22 +237,6 @@
# Python dotted path to the WSGI application used by Django's runserver.
WSGI_APPLICATION = 'toastermain.wsgi.application'
-TEMPLATE_DIRS = (
- # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
- # Always use forward slashes, even on Windows.
- # Don't forget to use absolute paths, not relative paths.
-)
-
-TEMPLATE_CONTEXT_PROCESSORS = ('django.contrib.auth.context_processors.auth',
- 'django.core.context_processors.debug',
- 'django.core.context_processors.i18n',
- 'django.core.context_processors.media',
- 'django.core.context_processors.static',
- 'django.core.context_processors.tz',
- 'django.contrib.messages.context_processors.messages',
- "django.core.context_processors.request",
- 'toastergui.views.managedcontextprocessor',
- )
INSTALLED_APPS = (
'django.contrib.auth',
@@ -348,10 +366,4 @@
#
-class InvalidString(str):
- def __mod__(self, other):
- from django.template.base import TemplateSyntaxError
- raise TemplateSyntaxError(
- "Undefined variable or unknown value for: \"%s\"" % other)
-TEMPLATE_STRING_IF_INVALID = InvalidString("%s")
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastermain/urls.py b/import-layers/yocto-poky/bitbake/lib/toaster/toastermain/urls.py
index 1f8599e..e2fb0ae 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastermain/urls.py
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastermain/urls.py
@@ -19,9 +19,10 @@
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-from django.conf.urls import patterns, include, url
-from django.views.generic import RedirectView
+from django.conf.urls import include, url
+from django.views.generic import RedirectView, TemplateView
from django.views.decorators.cache import never_cache
+import bldcollector.views
import logging
@@ -31,7 +32,7 @@
from django.contrib import admin
admin.autodiscover()
-urlpatterns = patterns('',
+urlpatterns = [
# Examples:
# url(r'^toaster/', include('toaster.foo.urls')),
@@ -42,11 +43,13 @@
# This is here to maintain backward compatibility and will be deprecated
# in the future.
- url(r'^orm/eventfile$', 'bldcollector.views.eventfile'),
+ url(r'^orm/eventfile$', bldcollector.views.eventfile),
+
+ url(r'^health$', TemplateView.as_view(template_name="health.html"), name='Toaster Health'),
# if no application is selected, we have the magic toastergui app here
url(r'^$', never_cache(RedirectView.as_view(url='/toastergui/', permanent=True))),
-)
+]
import toastermain.settings