diff --git a/import-layers/yocto-poky/bitbake/LICENSE b/import-layers/yocto-poky/bitbake/LICENSE
index 5d4a4c2..7d4e5f4 100644
--- a/import-layers/yocto-poky/bitbake/LICENSE
+++ b/import-layers/yocto-poky/bitbake/LICENSE
@@ -15,3 +15,5 @@
 * QUnit is redistributed under the MIT license.
 
 * Font Awesome fonts redistributed under the SIL Open Font License 1.1
+
+* simplediff is distributed under the zlib license.
diff --git a/import-layers/yocto-poky/bitbake/bin/bitbake b/import-layers/yocto-poky/bitbake/bin/bitbake
index 2a4fc72..9f5c2d4 100755
--- a/import-layers/yocto-poky/bitbake/bin/bitbake
+++ b/import-layers/yocto-poky/bitbake/bin/bitbake
@@ -38,7 +38,7 @@
 if sys.getfilesystemencoding() != "utf-8":
     sys.exit("Please use a locale setting which supports utf-8.\nPython can't change the filesystem locale after loading so we need a utf-8 when python starts or things won't work.")
 
-__version__ = "1.32.0"
+__version__ = "1.34.0"
 
 if __name__ == "__main__":
     if __version__ != bb.__version__:
diff --git a/import-layers/yocto-poky/bitbake/bin/bitbake-diffsigs b/import-layers/yocto-poky/bitbake/bin/bitbake-diffsigs
index 527d2c7..eb2f859 100755
--- a/import-layers/yocto-poky/bitbake/bin/bitbake-diffsigs
+++ b/import-layers/yocto-poky/bitbake/bin/bitbake-diffsigs
@@ -3,7 +3,7 @@
 # bitbake-diffsigs
 # BitBake task signature data comparison utility
 #
-# Copyright (C) 2012-2013 Intel Corporation
+# Copyright (C) 2012-2013, 2017 Intel Corporation
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -22,7 +22,7 @@
 import sys
 import warnings
 import fnmatch
-import optparse
+import argparse
 import logging
 import pickle
 
@@ -30,29 +30,13 @@
 
 import bb.tinfoil
 import bb.siggen
+import bb.msg
 
-def logger_create(name, output=sys.stderr):
-    logger = logging.getLogger(name)
-    console = logging.StreamHandler(output)
-    format = bb.msg.BBLogFormatter("%(levelname)s: %(message)s")
-    if output.isatty():
-        format.enable_color()
-    console.setFormatter(format)
-    logger.addHandler(console)
-    logger.setLevel(logging.INFO)
-    return logger
+logger = bb.msg.logger_create('bitbake-diffsigs')
 
-logger = logger_create('bitbake-diffsigs')
-
-def find_compare_task(bbhandler, pn, taskname):
+def find_compare_task(bbhandler, pn, taskname, sig1=None, sig2=None, color=False):
     """ Find the most recent signature files for the specified PN/task and compare them """
 
-    def get_hashval(siginfo):
-        if siginfo.endswith('.siginfo'):
-            return siginfo.rpartition(':')[2].partition('_')[0]
-        else:
-            return siginfo.rpartition('.')[2]
-
     if not hasattr(bb.siggen, 'find_siginfo'):
         logger.error('Metadata does not support finding signature data files')
         sys.exit(1)
@@ -60,79 +44,119 @@
     if not taskname.startswith('do_'):
         taskname = 'do_%s' % taskname
 
-    filedates = bb.siggen.find_siginfo(pn, taskname, None, bbhandler.config_data)
-    latestfiles = sorted(filedates.keys(), key=lambda f: filedates[f])[-3:]
-    if not latestfiles:
-        logger.error('No sigdata files found matching %s %s' % (pn, taskname))
-        sys.exit(1)
-    elif len(latestfiles) < 2:
-        logger.error('Only one matching sigdata file found for the specified task (%s %s)' % (pn, taskname))
-        sys.exit(1)
+    if sig1 and sig2:
+        sigfiles = bb.siggen.find_siginfo(pn, taskname, [sig1, sig2], bbhandler.config_data)
+        if len(sigfiles) == 0:
+            logger.error('No sigdata files found matching %s %s matching either %s or %s' % (pn, taskname, sig1, sig2))
+            sys.exit(1)
+        elif not sig1 in sigfiles:
+            logger.error('No sigdata files found matching %s %s with signature %s' % (pn, taskname, sig1))
+            sys.exit(1)
+        elif not sig2 in sigfiles:
+            logger.error('No sigdata files found matching %s %s with signature %s' % (pn, taskname, sig2))
+            sys.exit(1)
+        latestfiles = [sigfiles[sig1], sigfiles[sig2]]
     else:
-        # It's possible that latestfiles contain 3 elements and the first two have the same hash value.
-        # In this case, we delete the second element.
-        # The above case is actually the most common one. Because we may have sigdata file and siginfo
-        # file having the same hash value. Comparing such two files makes no sense.
-        if len(latestfiles) == 3:
-            hash0 = get_hashval(latestfiles[0])
-            hash1 = get_hashval(latestfiles[1])
-            if hash0 == hash1:
-                latestfiles.pop(1)
+        filedates = bb.siggen.find_siginfo(pn, taskname, None, bbhandler.config_data)
+        latestfiles = sorted(filedates.keys(), key=lambda f: filedates[f])[-3:]
+        if not latestfiles:
+            logger.error('No sigdata files found matching %s %s' % (pn, taskname))
+            sys.exit(1)
+        elif len(latestfiles) < 2:
+            logger.error('Only one matching sigdata file found for the specified task (%s %s)' % (pn, taskname))
+            sys.exit(1)
 
-        # Define recursion callback
-        def recursecb(key, hash1, hash2):
-            hashes = [hash1, hash2]
-            hashfiles = bb.siggen.find_siginfo(key, None, hashes, bbhandler.config_data)
+    # Define recursion callback
+    def recursecb(key, hash1, hash2):
+        hashes = [hash1, hash2]
+        hashfiles = bb.siggen.find_siginfo(key, None, hashes, bbhandler.config_data)
 
-            recout = []
-            if len(hashfiles) == 2:
-                out2 = bb.siggen.compare_sigfiles(hashfiles[hash1], hashfiles[hash2], recursecb)
-                recout.extend(list('  ' + l for l in out2))
-            else:
-                recout.append("Unable to find matching sigdata for %s with hashes %s or %s" % (key, hash1, hash2))
+        recout = []
+        if len(hashfiles) == 0:
+            recout.append("Unable to find matching sigdata for %s with hashes %s or %s" % (key, hash1, hash2))
+        elif not hash1 in hashfiles:
+            recout.append("Unable to find matching sigdata for %s with hash %s" % (key, hash1))
+        elif not hash2 in hashfiles:
+            recout.append("Unable to find matching sigdata for %s with hash %s" % (key, hash2))
+        else:
+            out2 = bb.siggen.compare_sigfiles(hashfiles[hash1], hashfiles[hash2], recursecb, color=color)
+            for change in out2:
+                for line in change.splitlines():
+                    recout.append('  ' + line)
 
-            return recout
+        return recout
 
-        # Recurse into signature comparison
-        output = bb.siggen.compare_sigfiles(latestfiles[0], latestfiles[1], recursecb)
-        if output:
-            print('\n'.join(output))
+    # Recurse into signature comparison
+    logger.debug("Signature file (previous): %s" % latestfiles[-2])
+    logger.debug("Signature file (latest): %s" % latestfiles[-1])
+    output = bb.siggen.compare_sigfiles(latestfiles[-2], latestfiles[-1], recursecb, color=color)
+    if output:
+        print('\n'.join(output))
     sys.exit(0)
 
 
 
-parser = optparse.OptionParser(
-    description = "Compares siginfo/sigdata files written out by BitBake",
-    usage = """
-  %prog -t recipename taskname
-  %prog sigdatafile1 sigdatafile2
-  %prog sigdatafile1""")
+parser = argparse.ArgumentParser(
+    description="Compares siginfo/sigdata files written out by BitBake")
 
-parser.add_option("-t", "--task",
-        help = "find the signature data files for last two runs of the specified task and compare them",
-        action="store", dest="taskargs", nargs=2, metavar='recipename taskname')
+parser.add_argument('-d', '--debug',
+                    help='Enable debug output',
+                    action='store_true')
 
-options, args = parser.parse_args(sys.argv)
+parser.add_argument('--color',
+        help='Colorize output (where %(metavar)s is %(choices)s)',
+        choices=['auto', 'always', 'never'], default='auto', metavar='color')
+
+parser.add_argument("-t", "--task",
+        help="find the signature data files for last two runs of the specified task and compare them",
+        action="store", dest="taskargs", nargs=2, metavar=('recipename', 'taskname'))
+
+parser.add_argument("-s", "--signature",
+        help="With -t/--task, specify the signatures to look for instead of taking the last two",
+        action="store", dest="sigargs", nargs=2, metavar=('fromsig', 'tosig'))
+
+parser.add_argument("sigdatafile1",
+        help="First signature file to compare (or signature file to dump, if second not specified). Not used when using -t/--task.",
+        action="store", nargs='?')
+
+parser.add_argument("sigdatafile2",
+        help="Second signature file to compare",
+        action="store", nargs='?')
+
+
+options = parser.parse_args()
+
+if options.debug:
+    logger.setLevel(logging.DEBUG)
+
+color = (options.color == 'always' or (options.color == 'auto' and sys.stdout.isatty()))
 
 if options.taskargs:
     with bb.tinfoil.Tinfoil() as tinfoil:
         tinfoil.prepare(config_only=True)
-        find_compare_task(tinfoil, options.taskargs[0], options.taskargs[1])
+        if options.sigargs:
+            find_compare_task(tinfoil, options.taskargs[0], options.taskargs[1], options.sigargs[0], options.sigargs[1], color=color)
+        else:
+            find_compare_task(tinfoil, options.taskargs[0], options.taskargs[1], color=color)
 else:
-    if len(args) == 1:
-        parser.print_help()
-    else:
-        try:
-            if len(args) == 2:
-                output = bb.siggen.dump_sigfile(sys.argv[1])
-            else:
-                output = bb.siggen.compare_sigfiles(sys.argv[1], sys.argv[2])
-        except IOError as e:
-            logger.error(str(e))
+    if options.sigargs:
+        logger.error('-s/--signature can only be used together with -t/--task')
+        sys.exit(1)
+    try:
+        if options.sigdatafile1 and options.sigdatafile2:
+            output = bb.siggen.compare_sigfiles(options.sigdatafile1, options.sigdatafile2, color=color)
+        elif options.sigdatafile1:
+            output = bb.siggen.dump_sigfile(options.sigdatafile1)
+        else:
+            logger.error('Must specify signature file(s) or -t/--task')
+            parser.print_help()
             sys.exit(1)
-        except (pickle.UnpicklingError, EOFError):
-            logger.error('Invalid signature data - ensure you are specifying sigdata/siginfo files')
-            sys.exit(1)
+    except IOError as e:
+        logger.error(str(e))
+        sys.exit(1)
+    except (pickle.UnpicklingError, EOFError):
+        logger.error('Invalid signature data - ensure you are specifying sigdata/siginfo files')
+        sys.exit(1)
 
-        if output:
-            print('\n'.join(output))
+    if output:
+        print('\n'.join(output))
diff --git a/import-layers/yocto-poky/bitbake/bin/bitbake-dumpsig b/import-layers/yocto-poky/bitbake/bin/bitbake-dumpsig
index 58ba1ca..95ebd93 100755
--- a/import-layers/yocto-poky/bitbake/bin/bitbake-dumpsig
+++ b/import-layers/yocto-poky/bitbake/bin/bitbake-dumpsig
@@ -27,39 +27,68 @@
 
 sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
 
+import bb.tinfoil
 import bb.siggen
+import bb.msg
 
-def logger_create(name, output=sys.stderr):
-    logger = logging.getLogger(name)
-    console = logging.StreamHandler(output)
-    format = bb.msg.BBLogFormatter("%(levelname)s: %(message)s")
-    if output.isatty():
-        format.enable_color()
-    console.setFormatter(format)
-    logger.addHandler(console)
-    logger.setLevel(logging.INFO)
-    return logger
+logger = bb.msg.logger_create('bitbake-dumpsig')
 
-logger = logger_create('bitbake-dumpsig')
+def find_siginfo_task(bbhandler, pn, taskname):
+    """ Find the most recent signature file for the specified PN/task """
+
+    if not hasattr(bb.siggen, 'find_siginfo'):
+        logger.error('Metadata does not support finding signature data files')
+        sys.exit(1)
+
+    if not taskname.startswith('do_'):
+        taskname = 'do_%s' % taskname
+
+    filedates = bb.siggen.find_siginfo(pn, taskname, None, bbhandler.config_data)
+    latestfiles = sorted(filedates.keys(), key=lambda f: filedates[f])[-1:]
+    if not latestfiles:
+        logger.error('No sigdata files found matching %s %s' % (pn, taskname))
+        sys.exit(1)
+
+    return latestfiles[0]
 
 parser = optparse.OptionParser(
     description = "Dumps siginfo/sigdata files written out by BitBake",
     usage = """
+  %prog -t recipename taskname
   %prog sigdatafile""")
 
+parser.add_option("-D", "--debug",
+        help = "enable debug",
+        action = "store_true", dest="debug", default = False)
+
+parser.add_option("-t", "--task",
+        help = "find the signature data file for the specified task",
+        action="store", dest="taskargs", nargs=2, metavar='recipename taskname')
+
 options, args = parser.parse_args(sys.argv)
 
-if len(args) == 1:
-    parser.print_help()
-else:
-    try:
-        output = bb.siggen.dump_sigfile(args[1])
-    except IOError as e:
-        logger.error(str(e))
-        sys.exit(1)
-    except (pickle.UnpicklingError, EOFError):
-        logger.error('Invalid signature data - ensure you are specifying a sigdata/siginfo file')
-        sys.exit(1)
+if options.debug:
+    logger.setLevel(logging.DEBUG)
 
-    if output:
-        print('\n'.join(output))
+if options.taskargs:
+    tinfoil = bb.tinfoil.Tinfoil()
+    tinfoil.prepare(config_only = True)
+    file = find_siginfo_task(tinfoil, options.taskargs[0], options.taskargs[1])
+    logger.debug("Signature file: %s" % file)
+elif len(args) == 1:
+    parser.print_help()
+    sys.exit(0)
+else:
+    file = args[1]
+
+try:
+    output = bb.siggen.dump_sigfile(file)
+except IOError as e:
+    logger.error(str(e))
+    sys.exit(1)
+except (pickle.UnpicklingError, EOFError):
+    logger.error('Invalid signature data - ensure you are specifying a sigdata/siginfo file')
+    sys.exit(1)
+
+if output:
+    print('\n'.join(output))
diff --git a/import-layers/yocto-poky/bitbake/bin/bitbake-layers b/import-layers/yocto-poky/bitbake/bin/bitbake-layers
index 946def2..2b05d28 100755
--- a/import-layers/yocto-poky/bitbake/bin/bitbake-layers
+++ b/import-layers/yocto-poky/bitbake/bin/bitbake-layers
@@ -24,43 +24,19 @@
 import os
 import sys
 import argparse
+import signal
 
 bindir = os.path.dirname(__file__)
 topdir = os.path.dirname(bindir)
 sys.path[0:0] = [os.path.join(topdir, 'lib')]
 
 import bb.tinfoil
+import bb.msg
 
-
-def tinfoil_init(parserecipes):
-    import bb.tinfoil
-    tinfoil = bb.tinfoil.Tinfoil(tracking=True)
-    tinfoil.prepare(not parserecipes)
-    tinfoil.logger.setLevel(logger.getEffectiveLevel())
-    return tinfoil
-
-
-def logger_create(name, output=sys.stderr):
-    logger = logging.getLogger(name)
-    loggerhandler = logging.StreamHandler(output)
-    loggerhandler.setFormatter(logging.Formatter("%(levelname)s: %(message)s"))
-    logger.addHandler(loggerhandler)
-    logger.setLevel(logging.INFO)
-    return logger
-
-def logger_setup_color(logger, color='auto'):
-    from bb.msg import BBLogFormatter
-    console = logging.StreamHandler(sys.stdout)
-    formatter = BBLogFormatter("%(levelname)s: %(message)s")
-    console.setFormatter(formatter)
-    logger.handlers = [console]
-    if color == 'always' or (color == 'auto' and console.stream.isatty()):
-        formatter.enable_color()
-
-
-logger = logger_create('bitbake-layers', sys.stdout)
+logger = bb.msg.logger_create('bitbake-layers', sys.stdout)
 
 def main():
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
     parser = argparse.ArgumentParser(
         description="BitBake layers utility",
         epilog="Use %(prog)s <subcommand> --help to get help on a specific command",
@@ -83,13 +59,17 @@
     elif global_args.quiet:
         logger.setLevel(logging.ERROR)
 
-    logger_setup_color(logger, global_args.color)
+    # Need to re-run logger_create with color argument
+    # (will be the same logger since it has the same name)
+    bb.msg.logger_create('bitbake-layers', output=sys.stdout, color=global_args.color)
 
     plugins = []
-    tinfoil = tinfoil_init(False)
+    tinfoil = bb.tinfoil.Tinfoil(tracking=True)
+    tinfoil.logger.setLevel(logger.getEffectiveLevel())
     try:
+        tinfoil.prepare(True)
         for path in ([topdir] +
-                    tinfoil.config_data.getVar('BBPATH', True).split(':')):
+                    tinfoil.config_data.getVar('BBPATH').split(':')):
             pluginpath = os.path.join(path, 'lib', 'bblayers')
             bb.utils.load_plugins(logger, plugins, pluginpath)
 
diff --git a/import-layers/yocto-poky/bitbake/bin/bitbake-worker b/import-layers/yocto-poky/bitbake/bin/bitbake-worker
index db3c4b1..ee2d622 100755
--- a/import-layers/yocto-poky/bitbake/bin/bitbake-worker
+++ b/import-layers/yocto-poky/bitbake/bin/bitbake-worker
@@ -95,6 +95,7 @@
             pass
         while (worker_queue_int or not worker_queue.empty()):
             try:
+                (_, ready, _) = select.select([], [worker_pipe], [], 1)
                 if not worker_queue.empty():
                     worker_queue_int = worker_queue_int + worker_queue.get()
                 written = os.write(worker_pipe, worker_queue_int)
@@ -135,7 +136,7 @@
     os.killpg(0, signal.SIGTERM)
     sys.exit()
 
-def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, appends, taskdepdata, quieterrors=False):
+def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, appends, taskdepdata, extraconfigdata, quieterrors=False, dry_run_exec=False):
     # We need to setup the environment BEFORE the fork, since
     # a fork() or exec*() activates PSEUDO...
 
@@ -151,8 +152,10 @@
         except TypeError:
              umask = taskdep['umask'][taskname]
 
+    dry_run = cfg.dry_run or dry_run_exec
+
     # We can't use the fakeroot environment in a dry run as it possibly hasn't been built
-    if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not cfg.dry_run:
+    if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not dry_run:
         envvars = (workerdata["fakerootenv"][fn] or "").split()
         for key, value in (var.split('=') for var in envvars):
             envbackup[key] = os.environ.get(key)
@@ -219,16 +222,21 @@
                 the_data = databuilder.mcdata[mc]
                 the_data.setVar("BB_WORKERCONTEXT", "1")
                 the_data.setVar("BB_TASKDEPDATA", taskdepdata)
+                if cfg.limited_deps:
+                    the_data.setVar("BB_LIMITEDDEPS", "1")
                 the_data.setVar("BUILDNAME", workerdata["buildname"])
                 the_data.setVar("DATE", workerdata["date"])
                 the_data.setVar("TIME", workerdata["time"])
+                for varname, value in extraconfigdata.items():
+                    the_data.setVar(varname, value)
+
                 bb.parse.siggen.set_taskdata(workerdata["sigdata"])
                 ret = 0
 
                 the_data = bb_cache.loadDataFull(fn, appends)
                 the_data.setVar('BB_TASKHASH', workerdata["runq_hash"][task])
 
-                bb.utils.set_process_name("%s:%s" % (the_data.getVar("PN", True), taskname.replace("do_", "")))
+                bb.utils.set_process_name("%s:%s" % (the_data.getVar("PN"), taskname.replace("do_", "")))
 
                 # exported_vars() returns a generator which *cannot* be passed to os.environ.update() 
                 # successfully. We also need to unset anything from the environment which shouldn't be there 
@@ -243,11 +251,11 @@
                     the_data.setVar(e, fakeenv[e])
                     the_data.setVarFlag(e, 'export', "1")
 
-                task_exports = the_data.getVarFlag(taskname, 'exports', True)
+                task_exports = the_data.getVarFlag(taskname, 'exports')
                 if task_exports:
                     for e in task_exports.split():
                         the_data.setVarFlag(e, 'export', '1')
-                        v = the_data.getVar(e, True)
+                        v = the_data.getVar(e)
                         if v is not None:
                             os.environ[e] = v
 
@@ -259,7 +267,7 @@
                     logger.critical(traceback.format_exc())
                 os._exit(1)
             try:
-                if cfg.dry_run:
+                if dry_run:
                     return 0
                 return bb.build.exec_task(fn, taskname, the_data, cfg.profile)
             except:
@@ -328,6 +336,7 @@
         self.cookercfg = None
         self.databuilder = None
         self.data = None
+        self.extraconfigdata = None
         self.build_pids = {}
         self.build_pipes = {}
     
@@ -362,6 +371,7 @@
                     pass
             if len(self.queue):
                 self.handle_item(b"cookerconfig", self.handle_cookercfg)
+                self.handle_item(b"extraconfigdata", self.handle_extraconfigdata)
                 self.handle_item(b"workerdata", self.handle_workerdata)
                 self.handle_item(b"runtask", self.handle_runtask)
                 self.handle_item(b"finishnow", self.handle_finishnow)
@@ -369,9 +379,11 @@
                 self.handle_item(b"quit", self.handle_quit)
 
             for pipe in self.build_pipes:
-                self.build_pipes[pipe].read()
+                if self.build_pipes[pipe].input in ready:
+                    self.build_pipes[pipe].read()
             if len(self.build_pids):
-                self.process_waitpid()
+                while self.process_waitpid():
+                    continue
 
 
     def handle_item(self, item, func):
@@ -388,6 +400,9 @@
         self.databuilder.parseBaseConfiguration()
         self.data = self.databuilder.data
 
+    def handle_extraconfigdata(self, data):
+        self.extraconfigdata = pickle.loads(data)
+
     def handle_workerdata(self, data):
         self.workerdata = pickle.loads(data)
         bb.msg.loggerDefaultDebugLevel = self.workerdata["logdefaultdebug"]
@@ -410,10 +425,10 @@
         sys.exit(0)
 
     def handle_runtask(self, data):
-        fn, task, taskname, quieterrors, appends, taskdepdata = pickle.loads(data)
+        fn, task, taskname, quieterrors, appends, taskdepdata, dry_run_exec = pickle.loads(data)
         workerlog_write("Handling runtask %s %s %s\n" % (task, fn, taskname))
 
-        pid, pipein, pipeout = fork_off_task(self.cookercfg, self.data, self.databuilder, self.workerdata, fn, task, taskname, appends, taskdepdata, quieterrors)
+        pid, pipein, pipeout = fork_off_task(self.cookercfg, self.data, self.databuilder, self.workerdata, fn, task, taskname, appends, taskdepdata, self.extraconfigdata, quieterrors, dry_run_exec)
 
         self.build_pids[pid] = task
         self.build_pipes[pid] = runQueueWorkerPipe(pipein, pipeout)
@@ -426,9 +441,9 @@
         try:
             pid, status = os.waitpid(-1, os.WNOHANG)
             if pid == 0 or os.WIFSTOPPED(status):
-                return None
+                return False
         except OSError:
-            return None
+            return False
 
         workerlog_write("Exit code of %s for pid %s\n" % (status, pid))
 
@@ -447,6 +462,8 @@
 
         worker_fire_prepickled(b"<exitcode>" + pickle.dumps((task, status)) + b"</exitcode>")
 
+        return True
+
     def handle_finishnow(self, _):
         if self.build_pids:
             logger.info("Sending SIGTERM to remaining %s tasks", len(self.build_pids))
diff --git a/import-layers/yocto-poky/bitbake/bin/toaster b/import-layers/yocto-poky/bitbake/bin/toaster
index f92d38e..61a4a0f 100755
--- a/import-layers/yocto-poky/bitbake/bin/toaster
+++ b/import-layers/yocto-poky/bitbake/bin/toaster
@@ -24,6 +24,29 @@
         [webport] Set the development server (default: localhost:8000)
 "
 
+databaseCheck()
+{
+    retval=0
+    # you can always add a superuser later via
+    # ../bitbake/lib/toaster/manage.py createsuperuser --username=<ME>
+    $MANAGE migrate --noinput || retval=1
+
+    if [ $retval -eq 1 ]; then
+        echo "Failed migrations, aborting system start" 1>&2
+        return $retval
+    fi
+    # Make sure that checksettings can pick up any value for TEMPLATECONF
+    export TEMPLATECONF
+    $MANAGE checksettings --traceback || retval=1
+
+    if [ $retval -eq 1 ]; then
+        printf "\nError while checking settings; aborting\n"
+        return $retval
+    fi
+
+    return $retval
+}
+
 webserverKillAll()
 {
     local pidfile
@@ -48,22 +71,9 @@
     fi
 
     retval=0
-    # you can always add a superuser later via
-    # ../bitbake/lib/toaster/manage.py createsuperuser --username=<ME>
-    $MANAGE migrate --noinput || retval=1
 
-    if [ $retval -eq 1 ]; then
-        echo "Failed migrations, aborting system start" 1>&2
-        return $retval
-    fi
-    # Make sure that checksettings can pick up any value for TEMPLATECONF
-    export TEMPLATECONF
-    $MANAGE checksettings --traceback || retval=1
-
-    if [ $retval -eq 1 ]; then
-        printf "\nError while checking settings; aborting\n"
-        return $retval
-    fi
+    # check the database
+    databaseCheck || return 1
 
     echo "Starting webserver..."
 
@@ -111,7 +121,7 @@
     if ! sed -n "$exp" $reqfile | python3 - ; then
         req=`grep ^Django $reqfile`
         echo "This program needs $req"
-        echo "Please install with pip install -r $reqfile"
+        echo "Please install with pip3 install -r $reqfile"
         return 2
     fi
 
@@ -151,14 +161,7 @@
 
 unset OE_ROOT
 
-# this defines the dir toaster will use for
-# 1) clones of layers (in _toaster_clones )
-# 2) the build dir (in build)
-# 3) the sqlite db if that is being used.
-# 4) pid's we need to clean up on exit/shutdown
-# note: for future. in order to make this an arbitrary directory, we need to
-# make sure that the toaster.sqlite file doesn't default to `pwd` like it currently does.
-export TOASTER_DIR=`pwd`
+
 
 WEBSERVER=1
 ADDR_PORT="localhost:8000"
@@ -214,10 +217,8 @@
 # 2) the build dir (in build)
 # 3) the sqlite db if that is being used.
 # 4) pid's we need to clean up on exit/shutdown
-# note: for future. in order to make this an arbitrary directory, we need to
-# make sure that the toaster.sqlite file doesn't default to `pwd`
-# like it currently does.
 export TOASTER_DIR=`dirname $BUILDDIR`
+export BB_ENV_EXTRAWHITE="$BB_ENV_EXTRAWHITE TOASTER_DIR"
 
 # Determine the action. If specified by arguments, fine, if not, toggle it
 if [ "$CMD" = "start" ] ; then
@@ -249,12 +250,25 @@
         line='INHERIT+="toaster buildhistory"'
         grep -q "$line" $conf || echo $line >> $conf
 
+        if [ $WEBSERVER -eq 0 ] ; then
+            # Do not update the database for "noweb" unless
+            # it does not yet exist
+            if [ ! -f "$TOASTER_DIR/toaster.sqlite" ] ; then
+                if ! databaseCheck; then
+                    echo "Failed ${CMD}."
+                  return 4
+                fi
+            fi
+        fi
         if [ $WEBSERVER -gt 0 ] && ! webserverStartAll; then
             echo "Failed ${CMD}."
             return 4
         fi
         export BITBAKE_UI='toasterui'
-        $MANAGE runbuilds & echo $! >${BUILDDIR}/.runbuilds.pid
+        $MANAGE runbuilds \
+           </dev/null >>${BUILDDIR}/toaster_runbuilds.log 2>&1 \
+           & echo $! >${BUILDDIR}/.runbuilds.pid
+
         # set fail safe stop system on terminal exit
         trap stop_system SIGHUP
         echo "Successful ${CMD}."
diff --git a/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml
index 2a3340b..d1ce43e 100644
--- a/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml
+++ b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml
@@ -38,7 +38,7 @@
             The code to execute the first part of this process, a fetch,
             looks something like the following:
             <literallayout class='monospaced'>
-     src_uri = (d.getVar('SRC_URI', True) or "").split()
+     src_uri = (d.getVar('SRC_URI') or "").split()
      fetcher = bb.fetch2.Fetch(src_uri, d)
      fetcher.download()
             </literallayout>
@@ -52,7 +52,7 @@
         <para>
             The instantiation of the fetch class is usually followed by:
             <literallayout class='monospaced'>
-     rootdir = l.getVar('WORKDIR', True)
+     rootdir = l.getVar('WORKDIR')
      fetcher.unpack(rootdir)
             </literallayout>
             This code unpacks the downloaded files to the
@@ -268,15 +268,6 @@
                 <link linkend='var-FILESPATH'><filename>FILESPATH</filename></link>
                 variable is used in the same way
                 <filename>PATH</filename> is used to find executables.
-                Failing that,
-                <link linkend='var-FILESDIR'><filename>FILESDIR</filename></link>
-                is used to find the appropriate relative file.
-                <note>
-                    <filename>FILESDIR</filename> is deprecated and can
-                    be replaced with <filename>FILESPATH</filename>.
-                    Because <filename>FILESDIR</filename> is likely to be
-                    removed, you should not use this variable in any new code.
-                </note>
                 If the file cannot be found, it is assumed that it is available in
                 <link linkend='var-DL_DIR'><filename>DL_DIR</filename></link>
                 by the time the <filename>download()</filename> method is called.
@@ -385,7 +376,8 @@
                 The supported parameters are as follows:
                 <itemizedlist>
                     <listitem><para><emphasis>"method":</emphasis>
-                        The protocol over which to communicate with the CVS server.
+                        The protocol over which to communicate with the CVS
+                        server.
                         By default, this protocol is "pserver".
                         If "method" is set to "ext", BitBake examines the
                         "rsh" parameter and sets <filename>CVS_RSH</filename>.
@@ -469,25 +461,29 @@
                         You can think of this parameter as the top-level
                         directory of the repository data you want.
                         </para></listitem>
+                    <listitem><para><emphasis>"path_spec":</emphasis>
+                        A specific directory in which to checkout the
+                        specified svn module.
+                        </para></listitem>
                     <listitem><para><emphasis>"protocol":</emphasis>
                         The protocol to use, which defaults to "svn".
-                        Other options are "svn+ssh" and "rsh".
-                        For "rsh", the "rsh" parameter is also used.
+                        If "protocol" is set to "svn+ssh", the "ssh"
+                        parameter is also used.
                         </para></listitem>
                     <listitem><para><emphasis>"rev":</emphasis>
                         The revision of the source code to checkout.
                         </para></listitem>
-                    <listitem><para><emphasis>"date":</emphasis>
-                        The date of the source code to checkout.
-                        Specific revisions are generally much safer to checkout
-                        rather than by date as they do not involve timezones
-                        (e.g. they are much more deterministic).
-                        </para></listitem>
                     <listitem><para><emphasis>"scmdata":</emphasis>
                         Causes the “.svn” directories to be available during
                         compile-time when set to "keep".
                         By default, these directories are removed.
                         </para></listitem>
+                    <listitem><para><emphasis>"ssh":</emphasis>
+                        An optional parameter used when "protocol" is set
+                        to "svn+ssh".
+                        You can use this parameter to specify the ssh
+                        program used by svn.
+                        </para></listitem>
                     <listitem><para><emphasis>"transportuser":</emphasis>
                         When required, sets the username for the transport.
                         By default, this parameter is empty.
@@ -496,10 +492,11 @@
                         command.
                         </para></listitem>
                 </itemizedlist>
-                Following are two examples using svn:
+                Following are three examples using svn:
                 <literallayout class='monospaced'>
-     SRC_URI = "svn://svn.oe.handhelds.org/svn;module=vip;proto=http;rev=667"
-     SRC_URI = "svn://svn.oe.handhelds.org/svn/;module=opie;proto=svn+ssh;date=20060126"
+     SRC_URI = "svn://myrepos/proj1;module=vip;protocol=http;rev=667"
+     SRC_URI = "svn://myrepos/proj1;module=opie;protocol=svn+ssh"
+     SRC_URI = "svn://myrepos/proj1;module=trunk;protocol=http;path_spec=${MY_DIR}/proj1"
                 </literallayout>
             </para>
         </section>
diff --git a/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-hello.xml b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-hello.xml
index 8b7edbf..2685c0e 100644
--- a/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-hello.xml
+++ b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-hello.xml
@@ -134,7 +134,7 @@
                     <ulink url="http://www.mail-archive.com/yocto@yoctoproject.org/msg09379.html">Mailing List post - The BitBake equivalent of "Hello, World!"</ulink>
                     </para></listitem>
                 <listitem><para>
-                    <ulink url="http://hambedded.org/blog/2012/11/24/from-bitbake-hello-world-to-an-image/">Hambedded Linux blog post - From Bitbake Hello World to an Image</ulink>
+                    <ulink url="https://web.archive.org/web/20150325165911/http://hambedded.org/blog/2012/11/24/from-bitbake-hello-world-to-an-image/">Hambedded Linux blog post - From Bitbake Hello World to an Image</ulink>
                     </para></listitem>
             </itemizedlist>
         </note>
@@ -269,7 +269,7 @@
                 and define some key BitBake variables.
                 For more information on the <filename>bitbake.conf</filename>,
                 see
-                <ulink url='http://hambedded.org/blog/2012/11/24/from-bitbake-hello-world-to-an-image/#an-overview-of-bitbakeconf'></ulink>
+                <ulink url='https://web.archive.org/web/20150325165911/http://hambedded.org/blog/2012/11/24/from-bitbake-hello-world-to-an-image/#an-overview-of-bitbakeconf'></ulink>
                 </para>
                 <para>Use the following commands to create the <filename>conf</filename>
                 directory in the project directory:
@@ -354,7 +354,7 @@
                 supporting.
                 For more information on the <filename>base.bbclass</filename> file,
                 you can look at
-                <ulink url='http://hambedded.org/blog/2012/11/24/from-bitbake-hello-world-to-an-image/#tasks'></ulink>.
+                <ulink url='https://web.archive.org/web/20150325165911/http://hambedded.org/blog/2012/11/24/from-bitbake-hello-world-to-an-image/#tasks'></ulink>.
                 </para></listitem>
             <listitem><para><emphasis>Run Bitbake:</emphasis>
                 After making sure that the <filename>classes/base.bbclass</filename>
@@ -376,7 +376,7 @@
                 Thus, this example creates and uses a layer called "mylayer".
                 <note>
                     You can find additional information on adding a layer at
-                    <ulink url='http://hambedded.org/blog/2012/11/24/from-bitbake-hello-world-to-an-image/#adding-an-example-layer'></ulink>.
+                    <ulink url='https://web.archive.org/web/20150325165911/http://hambedded.org/blog/2012/11/24/from-bitbake-hello-world-to-an-image/#adding-an-example-layer'></ulink>.
                 </note>
                 </para>
                 <para>Minimally, you need a recipe file and a layer configuration
diff --git a/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-intro.xml b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-intro.xml
index 4d58dc4..ca7f724 100644
--- a/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-intro.xml
+++ b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-intro.xml
@@ -504,9 +504,19 @@
                              Read the specified file before bitbake.conf.
        -R POSTFILE, --postread=POSTFILE
                              Read the specified file after bitbake.conf.
-       -v, --verbose         Output more log message data to the terminal.
-       -D, --debug           Increase the debug level. You can specify this more
-                             than once.
+       -v, --verbose         Enable tracing of shell tasks (with 'set -x').
+                             Also print bb.note(...) messages to stdout (in
+                             addition to writing them to ${T}/log.do_&lt;task&gt;).
+       -D, --debug           Increase the debug level. You can specify this
+                             more than once. -D sets the debug level to 1,
+                             where only bb.debug(1, ...) messages are printed
+                             to stdout; -DD sets the debug level to 2, where
+                             both bb.debug(1, ...) and bb.debug(2, ...)
+                             messages are printed; etc. Without -D, no debug
+                             messages are printed. Note that -D only affects
+                             output to stdout. All debug messages are written
+                             to ${T}/log.do_taskname, regardless of the debug
+                             level.
        -n, --dry-run         Don't execute, just go through the motions.
        -S SIGNATURE_HANDLER, --dump-signatures=SIGNATURE_HANDLER
                              Dump out the signature construction information, with
@@ -529,8 +539,8 @@
        -l DEBUG_DOMAINS, --log-domains=DEBUG_DOMAINS
                              Show debug logging for the specified logging domains
        -P, --profile         Profile the command and save reports.
-       -u UI, --ui=UI        The user interface to use (depexp, goggle, hob, knotty
-                             or ncurses - default knotty).
+       -u UI, --ui=UI        The user interface to use (taskexp, knotty or
+                             ncurses - default knotty).
        -t SERVERTYPE, --servertype=SERVERTYPE
                              Choose which server type to use (process or xmlrpc -
                              default process).
@@ -543,6 +553,7 @@
        -B BIND, --bind=BIND  The name/address for the bitbake server to bind to.
        --no-setscene         Do not run any setscene tasks. sstate will be ignored
                              and everything needed, built.
+       --setscene-only       Only run setscene tasks, don't run any real tasks.
        --remote-server=REMOTE_SERVER
                              Connect to the specified server.
        -m, --kill-server     Terminate the remote server.
@@ -665,21 +676,21 @@
                 </para>
 
                 <para>
-                    When you generate a dependency graph, BitBake writes four files
+                    When you generate a dependency graph, BitBake writes three files
                     to the current working directory:
                     <itemizedlist>
-                        <listitem><para><emphasis><filename>package-depends.dot</filename>:</emphasis>
-                            Shows BitBake's knowledge of dependencies between
-                            runtime targets.
+                        <listitem><para>
+                            <emphasis><filename>recipe-depends.dot</filename>:</emphasis>
+                            Shows dependencies between recipes (i.e. a collapsed version of
+                            <filename>task-depends.dot</filename>).
                             </para></listitem>
-                        <listitem><para><emphasis><filename>pn-depends.dot</filename>:</emphasis>
-                            Shows dependencies between build-time targets
-                            (i.e. recipes).
-                            </para></listitem>
-                        <listitem><para><emphasis><filename>task-depends.dot</filename>:</emphasis>
+                        <listitem><para>
+                            <emphasis><filename>task-depends.dot</filename>:</emphasis>
                             Shows dependencies between tasks.
+                            These dependencies match BitBake's internal task execution list.
                             </para></listitem>
-                        <listitem><para><emphasis><filename>pn-buildlist</filename>:</emphasis>
+                        <listitem><para>
+                            <emphasis><filename>pn-buildlist</filename>:</emphasis>
                             Shows a simple list of targets that are to be built.
                             </para></listitem>
                     </itemizedlist>
diff --git a/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.xml b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.xml
index 71bb25b..1d1e5b3 100644
--- a/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.xml
+++ b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.xml
@@ -61,6 +61,48 @@
             </para>
         </section>
 
+        <section id='line-joining'>
+            <title>Line Joining</title>
+
+            <para>
+                Outside of
+                <link linkend='functions'>functions</link>, BitBake joins
+                any line ending in a backslash character ("\")
+                with the following line before parsing statements.
+                The most common use for the "\" character is to split variable
+                assignments over multiple lines, as in the following example:
+                <literallayout class='monospaced'>
+     FOO = "bar \
+            baz \
+            qaz"
+                </literallayout>
+                Both the "\" character and the newline character
+                that follow it are removed when joining lines.
+                Thus, no newline characters end up in the value of
+                <filename>FOO</filename>.
+            </para>
+
+            <para>
+                Consider this additional example where the two
+                assignments both assign "barbaz" to
+                <filename>FOO</filename>:
+                <literallayout class='monospaced'>
+     FOO = "barbaz"
+
+     FOO = "bar\
+     baz"
+                </literallayout>
+                <note>
+                    BitBake does not interpret escape sequences like
+                    "\n" in variable values.
+                    For these to have an effect, the value must be passed
+                    to some utility that interprets escape sequences,
+                    such as <filename>printf</filename> or
+                    <filename>echo -n</filename>.
+                </note>
+            </para>
+        </section>
+
         <section id='variable-expansion'>
             <title>Variable Expansion</title>
 
@@ -463,14 +505,14 @@
             <title>Unseting variables</title>
 
             <para>
-                It is possible to completely remove a variable or a variable flag 
+                It is possible to completely remove a variable or a variable flag
                 from BitBake's internal data dictionary by using the "unset" keyword.
                 Here is an example:
                 <literallayout class='monospaced'>
         unset DATE
         unset do_fetch[noexec]
                 </literallayout>
-                These two statements remove the <filename>DATE</filename> and the 
+                These two statements remove the <filename>DATE</filename> and the
                 <filename>do_fetch[noexec]</filename> flag.
             </para>
 
@@ -1165,7 +1207,7 @@
                 <literallayout class='monospaced'>
      python some_python_function () {
          d.setVar("TEXT", "Hello World")
-         print d.getVar("TEXT", True)
+         print d.getVar("TEXT")
      }
                 </literallayout>
                 Because the Python "bb" and "os" modules are already
@@ -1180,7 +1222,7 @@
                     to freely set variable values to expandable expressions
                     without having them expanded prematurely.
                     If you do wish to expand a variable within a Python
-                    function, use <filename>d.getVar("X", True)</filename>.
+                    function, use <filename>d.getVar("X")</filename>.
                     Or, for more complicated expressions, use
                     <filename>d.expand()</filename>.
                 </note>
@@ -1232,7 +1274,7 @@
                 Here is an example:
                 <literallayout class='monospaced'>
      def get_depends(d):
-         if d.getVar('SOMECONDITION', True):
+         if d.getVar('SOMECONDITION'):
              return "dependencywithcond"
          else:
              return "dependency"
@@ -1367,7 +1409,7 @@
                 based on the value of  another variable:
                 <literallayout class='monospaced'>
      python () {
-         if d.getVar('SOMEVAR', True) == 'value':
+         if d.getVar('SOMEVAR') == 'value':
              d.setVar('ANOTHERVAR', 'value2')
      }
                 </literallayout>
@@ -1942,128 +1984,194 @@
         <title>Events</title>
 
         <para>
-            BitBake allows installation of event handlers within
-            recipe and class files.
-            Events are triggered at certain points during operation,
-            such as the beginning of an operation against a given recipe
-            (<filename>*.bb</filename> file), the start of a given task,
-            task failure, task success, and so forth.
+            BitBake allows installation of event handlers within recipe
+            and class files.
+            Events are triggered at certain points during operation, such
+            as the beginning of operation against a given recipe
+            (i.e. <filename>*.bb</filename>), the start of a given task,
+            a task failure, a task success, and so forth.
             The intent is to make it easy to do things like email
-            notification on build failure.
+            notification on build failures.
         </para>
 
         <para>
-            Following is an example event handler that
-            prints the name of the event and the content of
-            the <filename>FILE</filename> variable:
+            Following is an example event handler that prints the name
+            of the event and the content of the
+            <filename>FILE</filename> variable:
             <literallayout class='monospaced'>
      addhandler myclass_eventhandler
      python myclass_eventhandler() {
          from bb.event import getName
-         from bb import data
          print("The name of the Event is %s" % getName(e))
-         print("The file we run for is %s" % data.getVar('FILE', e.data, True))
+         print("The file we run for is %s" % d.getVar('FILE'))
      }
+     myclass_eventhandler[eventmask] = "bb.event.BuildStarted bb.event.BuildCompleted"
             </literallayout>
-            This event handler gets called every time an event is
-            triggered.
-            A global variable "<filename>e</filename>" is defined and
-            "<filename>e.data</filename>" contains an instance of
-            "<filename>bb.data</filename>".
-            With the <filename>getName(e)</filename> method, one can get
+            In the previous example, an eventmask has been set so that
+            the handler only sees the "BuildStarted" and "BuildCompleted"
+            events.
+            This event handler gets called every time an event matching
+            the eventmask is triggered.
+            A global variable "e" is defined, which represents the current
+            event.
+            With the <filename>getName(e)</filename> method, you can get
             the name of the triggered event.
+            The global datastore is available as "d".
+            In legacy code, you might see "e.data" used to get the datastore.
+            However, realize that "e.data" is deprecated and you should use
+            "d" going forward.
         </para>
 
         <para>
-            Because you probably are only interested in a subset of events,
-            you would likely use the <filename>[eventmask]</filename> flag
-            for your event handler to be sure that only certain events
-            trigger the handler.
-            Given the previous example, suppose you only wanted the
-            <filename>bb.build.TaskFailed</filename> event to trigger that
-            event handler.
-            Use the flag as follows:
-            <literallayout class='monospaced'>
-     addhandler myclass_eventhandler
-     myclass_eventhandler[eventmask] = "bb.build.TaskFailed"
-     python myclass_eventhandler() {
-         from bb.event import getName
-         from bb import data
-         print("The name of the Event is %s" % getName(e))
-         print("The file we run for is %s" % data.getVar('FILE', e.data, True))
-     }
-            </literallayout>
+            The context of the datastore is appropriate to the event
+            in question.
+            For example, "BuildStarted" and "BuildCompleted" events run
+            before any tasks are executed so would be in the global
+            configuration datastore namespace.
+            No recipe-specific metadata exists in that namespace.
+            The "BuildStarted" and "buildCompleted" events also run in
+            the main cooker/server process rather than any worker context.
+            Thus, any changes made to the datastore would be seen by other
+            cooker/server events within the current build but not seen
+            outside of that build or in any worker context.
+            Task events run in the actual tasks in question consequently
+            have recipe-specific and task-specific contents.
+            These events run in the worker context and are discarded at
+            the end of task execution.
         </para>
 
         <para>
-            During a standard build, the following common events might occur:
+            During a standard build, the following common events might
+            occur.
+            The following events are the most common kinds of events that
+            most metadata might have an interest in viewing:
             <itemizedlist>
                 <listitem><para>
-                    <filename>bb.event.ConfigParsed()</filename>
+                    <filename>bb.event.ConfigParsed()</filename>:
+                    Fired when the base configuration; which consists of
+                    <filename>bitbake.conf</filename>,
+                    <filename>base.bbclass</filename> and any global
+                    <filename>INHERIT</filename> statements; has been parsed.
+                    You can see multiple such events when each of the
+                    workers parse the base configuration or if the server
+                    changes configuration and reparses.
+                    Any given datastore only has one such event executed
+                    against it, however.
+                    If
+                    <link linkende='var-BB_INVALIDCONF'><filename>BB_INVALIDCONF</filename></link>
+                    is set in the datastore by the event handler, the
+                    configuration is reparsed and a new event triggered,
+                    allowing the metadata to update configuration.
                     </para></listitem>
                 <listitem><para>
-                    <filename>bb.event.ParseStarted()</filename>
+                    <filename>bb.event.HeartbeatEvent()</filename>:
+                    Fires at regular time intervals of one second.
+                    You can configure the interval time using the
+                    <filename>BB_HEARTBEAT_EVENT</filename> variable.
+                    The event's "time" attribute is the
+                    <filename>time.time()</filename> value when the
+                    event is triggered.
+                    This event is useful for activities such as
+                    system state monitoring.
                     </para></listitem>
                 <listitem><para>
-                    <filename>bb.event.ParseProgress()</filename>
+                    <filename>bb.event.ParseStarted()</filename>:
+                    Fired when BitBake is about to start parsing recipes.
+                    This event's "total" attribute represents the number of
+                    recipes BitBake plans to parse.
                     </para></listitem>
                 <listitem><para>
-                    <filename>bb.event.ParseCompleted()</filename>
+                    <filename>bb.event.ParseProgress()</filename>:
+                    Fired as parsing progresses.
+                    This event's "current" attribute is the number of
+                    recipes parsed as well as the "total" attribute.
                     </para></listitem>
                 <listitem><para>
-                    <filename>bb.event.BuildStarted()</filename>
+                    <filename>bb.event.ParseCompleted()</filename>:
+                    Fired when parsing is complete.
+                    This event's "cached", "parsed", "skipped", "virtuals",
+                    "masked", and "errors" attributes provide statistics
+                    for the parsing results.
                     </para></listitem>
                 <listitem><para>
-                    <filename>bb.build.TaskStarted()</filename>
+                    <filename>bb.event.BuildStarted()</filename>:
+                    Fired when a new build starts.
                     </para></listitem>
                 <listitem><para>
-                    <filename>bb.build.TaskInvalid()</filename>
+                    <filename>bb.build.TaskStarted()</filename>:
+                    Fired when a task starts.
+                    This event's "taskfile" attribute points to the recipe
+                    from which the task originates.
+                    The "taskname" attribute, which is the task's name,
+                    includes the <filename>do_</filename> prefix, and the
+                    "logfile" attribute point to where the task's output is
+                    stored.
+                    Finally, the "time" attribute is the task's execution start
+                    time.
                     </para></listitem>
                 <listitem><para>
-                    <filename>bb.build.TaskFailedSilent()</filename>
+                    <filename>bb.build.TaskInvalid()</filename>:
+                    Fired if BitBake tries to execute a task that does not exist.
                     </para></listitem>
                 <listitem><para>
-                    <filename>bb.build.TaskFailed()</filename>
+                    <filename>bb.build.TaskFailedSilent()</filename>:
+                    Fired for setscene tasks that fail and should not be
+                    presented to the user verbosely.
                     </para></listitem>
                 <listitem><para>
-                    <filename>bb.build.TaskSucceeded()</filename>
+                    <filename>bb.build.TaskFailed()</filename>:
+                    Fired for normal tasks that fail.
                     </para></listitem>
                 <listitem><para>
-                    <filename>bb.event.BuildCompleted()</filename>
+                    <filename>bb.build.TaskSucceeded()</filename>:
+                    Fired when a task successfully completes.
                     </para></listitem>
                 <listitem><para>
-                    <filename>bb.cooker.CookerExit()</filename>
+                    <filename>bb.event.BuildCompleted()</filename>:
+                    Fired when a build finishes.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>bb.cooker.CookerExit()</filename>:
+                    Fired when the BitBake server/cooker shuts down.
+                    This event is usually only seen by the UIs as a
+                    sign they should also shutdown.
                     </para></listitem>
             </itemizedlist>
-            Here is a list of other events that occur based on specific requests
-            to the server:
+        </para>
+
+        <para>
+            This next list of example events occur based on specific
+            requests to the server.
+            These events are often used to communicate larger pieces of
+            information from the BitBake server to other parts of
+            BitBake such as user interfaces:
             <itemizedlist>
                 <listitem><para>
                     <filename>bb.event.TreeDataPreparationStarted()</filename>
                     </para></listitem>
                 <listitem><para>
-                    <filename>bb.event.TreeDataPreparationProgress</filename>
+                    <filename>bb.event.TreeDataPreparationProgress()</filename>
                     </para></listitem>
                 <listitem><para>
-                    <filename>bb.event.TreeDataPreparationCompleted</filename>
+                    <filename>bb.event.TreeDataPreparationCompleted()</filename>
                     </para></listitem>
                 <listitem><para>
-                    <filename>bb.event.DepTreeGenerated</filename>
+                    <filename>bb.event.DepTreeGenerated()</filename>
                     </para></listitem>
                 <listitem><para>
-                    <filename>bb.event.CoreBaseFilesFound</filename>
+                    <filename>bb.event.CoreBaseFilesFound()</filename>
                     </para></listitem>
                 <listitem><para>
-                    <filename>bb.event.ConfigFilePathFound</filename>
+                    <filename>bb.event.ConfigFilePathFound()</filename>
                     </para></listitem>
                 <listitem><para>
-                    <filename>bb.event.FilesMatchingFound</filename>
+                    <filename>bb.event.FilesMatchingFound()</filename>
                     </para></listitem>
                 <listitem><para>
-                    <filename>bb.event.ConfigFilesFound</filename>
+                    <filename>bb.event.ConfigFilesFound()</filename>
                     </para></listitem>
                 <listitem><para>
-                    <filename>bb.event.TargetsTreeGenerated</filename>
+                    <filename>bb.event.TargetsTreeGenerated()</filename>
                     </para></listitem>
             </itemizedlist>
         </para>
diff --git a/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.xml b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.xml
index 66d8f84..0e89bf2 100644
--- a/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.xml
+++ b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.xml
@@ -1539,24 +1539,6 @@
             </glossdef>
         </glossentry>
 
-        <glossentry id='var-FILESDIR'><glossterm>FILESDIR</glossterm>
-            <glossdef>
-                <para>
-                    Specifies directories BitBake uses when searching for
-                    patches and files.
-                    The "local" fetcher module uses these directories when
-                    handling <filename>file://</filename> URLs if the file
-                    was not found using
-                    <link linkend='var-FILESPATH'><filename>FILESPATH</filename></link>.
-                    <note>
-                        The <filename>FILESDIR</filename> variable is
-                        deprecated and you should use
-                        <filename>FILESPATH</filename> in all new code.
-                    </note>
-                </para>
-            </glossdef>
-        </glossentry>
-
         <glossentry id='var-FILESPATH'><glossterm>FILESPATH</glossterm>
             <glossdef>
                 <para>
diff --git a/import-layers/yocto-poky/bitbake/doc/bitbake.1 b/import-layers/yocto-poky/bitbake/doc/bitbake.1
index a6c8d97..7fc1652 100644
--- a/import-layers/yocto-poky/bitbake/doc/bitbake.1
+++ b/import-layers/yocto-poky/bitbake/doc/bitbake.1
@@ -105,7 +105,7 @@
 profile the command and print a report
 .TP
 .B \-uUI, \-\-ui=UI
-User interface to use. Currently, hob, depexp, goggle or ncurses can be specified as UI.
+User interface to use. Currently, knotty, taskexp or ncurses can be specified as UI.
 .TP
 .B \-tSERVERTYPE, \-\-servertype=SERVERTYPE
 Choose which server to use, none, process or xmlrpc.
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/COW.py b/import-layers/yocto-poky/bitbake/lib/bb/COW.py
index 77a05cf..36ebbd9 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/COW.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/COW.py
@@ -213,11 +213,11 @@
     print()
 
     print("a", a)
-    for x in a.items():
+    for x in a.iteritems():
         print(x)
     print("--")
     print("b", b)
-    for x in b.items():
+    for x in b.iteritems():
         print(x)
     print()
 
@@ -225,11 +225,11 @@
     b['a'] = 'c'
 
     print("a", a)
-    for x in a.items():
+    for x in a.iteritems():
         print(x)
     print("--")
     print("b", b)
-    for x in b.items():
+    for x in b.iteritems():
         print(x)
     print()
 
@@ -244,22 +244,22 @@
     a['set'].add("o2")
 
     print("a", a)
-    for x in a['set'].values():
+    for x in a['set'].itervalues():
         print(x)
     print("--")
     print("b", b)
-    for x in b['set'].values():
+    for x in b['set'].itervalues():
         print(x)
     print()
 
     b['set'].add('o3')
 
     print("a", a)
-    for x in a['set'].values():
+    for x in a['set'].itervalues():
         print(x)
     print("--")
     print("b", b)
-    for x in b['set'].values():
+    for x in b['set'].itervalues():
         print(x)
     print()
 
@@ -269,7 +269,7 @@
     a['set2'].add("o2")
 
     print("a", a)
-    for x in a.items():
+    for x in a.iteritems():
         print(x)
     print("--")
     print("b", b)
@@ -289,7 +289,7 @@
         print("Yay - has_key with delete works!")
 
     print("a", a)
-    for x in a.items():
+    for x in a.iteritems():
         print(x)
     print("--")
     print("b", b)
@@ -300,7 +300,7 @@
     b.__revertitem__('b')
 
     print("a", a)
-    for x in a.items():
+    for x in a.iteritems():
         print(x)
     print("--")
     print("b", b)
@@ -310,7 +310,7 @@
 
     b.__revertitem__('dict')
     print("a", a)
-    for x in a.items():
+    for x in a.iteritems():
         print(x)
     print("--")
     print("b", b)
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/__init__.py b/import-layers/yocto-poky/bitbake/lib/bb/__init__.py
index f019d48..bfe0ca5 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/__init__.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/__init__.py
@@ -21,7 +21,7 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-__version__ = "1.32.0"
+__version__ = "1.34.0"
 
 import sys
 if sys.version_info < (3, 4, 0):
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/build.py b/import-layers/yocto-poky/bitbake/lib/bb/build.py
index b59a49b..0d0100a 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/build.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/build.py
@@ -91,14 +91,14 @@
 
     def __init__(self, t, logfile, d):
         self._task = t
-        self._package = d.getVar("PF", True)
-        self._mc = d.getVar("BB_CURRENT_MC", True)
-        self.taskfile = d.getVar("FILE", True)
+        self._package = d.getVar("PF")
+        self._mc = d.getVar("BB_CURRENT_MC")
+        self.taskfile = d.getVar("FILE")
         self.taskname = self._task
         self.logfile = logfile
         self.time = time.time()
         event.Event.__init__(self)
-        self._message = "recipe %s: task %s: %s" % (d.getVar("PF", True), t, self.getDisplayName())
+        self._message = "recipe %s: task %s: %s" % (d.getVar("PF"), t, self.getDisplayName())
 
     def getTask(self):
         return self._task
@@ -195,13 +195,13 @@
         oldcwd = None
 
     flags = d.getVarFlags(func)
-    cleandirs = flags.get('cleandirs')
+    cleandirs = flags.get('cleandirs') if flags else None
     if cleandirs:
         for cdir in d.expand(cleandirs).split():
             bb.utils.remove(cdir, True)
             bb.utils.mkdirhier(cdir)
 
-    if dirs is None:
+    if flags and dirs is None:
         dirs = flags.get('dirs')
         if dirs:
             dirs = d.expand(dirs).split()
@@ -227,17 +227,17 @@
     else:
         lockfiles = None
 
-    tempdir = d.getVar('T', True)
+    tempdir = d.getVar('T')
 
     # or func allows items to be executed outside of the normal
     # task set, such as buildhistory
-    task = d.getVar('BB_RUNTASK', True) or func
+    task = d.getVar('BB_RUNTASK') or func
     if task == func:
         taskfunc = task
     else:
         taskfunc = "%s.%s" % (task, func)
 
-    runfmt = d.getVar('BB_RUNFMT', True) or "run.{func}.{pid}"
+    runfmt = d.getVar('BB_RUNFMT') or "run.{func}.{pid}"
     runfn = runfmt.format(taskfunc=taskfunc, task=task, func=func, pid=os.getpid())
     runfile = os.path.join(tempdir, runfn)
     bb.utils.mkdirhier(os.path.dirname(runfile))
@@ -369,7 +369,7 @@
 
     cmd = runfile
     if d.getVarFlag(func, 'fakeroot', False):
-        fakerootcmd = d.getVar('FAKEROOT', True)
+        fakerootcmd = d.getVar('FAKEROOT')
         if fakerootcmd:
             cmd = [fakerootcmd, runfile]
 
@@ -378,7 +378,7 @@
     else:
         logfile = sys.stdout
 
-    progress = d.getVarFlag(func, 'progress', True)
+    progress = d.getVarFlag(func, 'progress')
     if progress:
         if progress == 'percent':
             # Use default regex
@@ -430,7 +430,7 @@
             else:
                 break
 
-    tempdir = d.getVar('T', True)
+    tempdir = d.getVar('T')
     fifopath = os.path.join(tempdir, 'fifo.%s' % os.getpid())
     if os.path.exists(fifopath):
         os.unlink(fifopath)
@@ -443,7 +443,7 @@
                 with open(os.devnull, 'r+') as stdin:
                     bb.process.run(cmd, shell=False, stdin=stdin, log=logfile, extrafiles=[(fifo,readfifo)])
             except bb.process.CmdError:
-                logfn = d.getVar('BB_LOGFILE', True)
+                logfn = d.getVar('BB_LOGFILE')
                 raise FuncFailed(func, logfn)
         finally:
             os.unlink(fifopath)
@@ -474,18 +474,18 @@
     logger.debug(1, "Executing task %s", task)
 
     localdata = _task_data(fn, task, d)
-    tempdir = localdata.getVar('T', True)
+    tempdir = localdata.getVar('T')
     if not tempdir:
         bb.fatal("T variable not set, unable to build")
 
     # Change nice level if we're asked to
-    nice = localdata.getVar("BB_TASK_NICE_LEVEL", True)
+    nice = localdata.getVar("BB_TASK_NICE_LEVEL")
     if nice:
         curnice = os.nice(0)
         nice = int(nice) - curnice
         newnice = os.nice(nice)
         logger.debug(1, "Renice to %s " % newnice)
-    ionice = localdata.getVar("BB_TASK_IONICE_LEVEL", True)
+    ionice = localdata.getVar("BB_TASK_IONICE_LEVEL")
     if ionice:
         try:
             cls, prio = ionice.split(".", 1)
@@ -496,7 +496,7 @@
     bb.utils.mkdirhier(tempdir)
 
     # Determine the logfile to generate
-    logfmt = localdata.getVar('BB_LOGFMT', True) or 'log.{task}.{pid}'
+    logfmt = localdata.getVar('BB_LOGFMT') or 'log.{task}.{pid}'
     logbase = logfmt.format(task=task, pid=os.getpid())
 
     # Document the order of the tasks...
@@ -563,6 +563,7 @@
 
     localdata.setVar('BB_LOGFILE', logfn)
     localdata.setVar('BB_RUNTASK', task)
+    localdata.setVar('BB_TASK_LOGGER', bblogger)
 
     flags = localdata.getVarFlags(task)
 
@@ -628,7 +629,7 @@
             quieterr = True
 
         if profile:
-            profname = "profile-%s.log" % (d.getVar("PN", True) + "-" + task)
+            profname = "profile-%s.log" % (d.getVar("PN") + "-" + task)
             try:
                 import cProfile as profile
             except:
@@ -668,9 +669,9 @@
         stamp = d.stamp[file_name]
         extrainfo = d.stamp_extrainfo[file_name].get(taskflagname) or ""
     else:
-        stamp = d.getVar('STAMP', True)
-        file_name = d.getVar('BB_FILENAME', True)
-        extrainfo = d.getVarFlag(taskflagname, 'stamp-extra-info', True) or ""
+        stamp = d.getVar('STAMP')
+        file_name = d.getVar('BB_FILENAME')
+        extrainfo = d.getVarFlag(taskflagname, 'stamp-extra-info') or ""
 
     if baseonly:
         return stamp
@@ -704,9 +705,9 @@
         stamp = d.stampclean[file_name]
         extrainfo = d.stamp_extrainfo[file_name].get(taskflagname) or ""
     else:
-        stamp = d.getVar('STAMPCLEAN', True)
-        file_name = d.getVar('BB_FILENAME', True)
-        extrainfo = d.getVarFlag(taskflagname, 'stamp-extra-info', True) or ""
+        stamp = d.getVar('STAMPCLEAN')
+        file_name = d.getVar('BB_FILENAME')
+        extrainfo = d.getVarFlag(taskflagname, 'stamp-extra-info') or ""
 
     if not stamp:
         return []
@@ -742,7 +743,7 @@
     # as it completes
     if not task.endswith("_setscene") and task != "do_setscene" and not file_name:
         stampbase = stamp_internal(task, d, None, True)
-        file_name = d.getVar('BB_FILENAME', True)
+        file_name = d.getVar('BB_FILENAME')
         bb.parse.siggen.dump_sigtask(file_name, task, stampbase, True)
 
 def del_stamp(task, d, file_name = None):
@@ -764,7 +765,7 @@
     if file_name:
         taintfn = d.stamp[file_name] + '.' + task + '.taint'
     else:
-        taintfn = d.getVar('STAMP', True) + '.' + task + '.taint'
+        taintfn = d.getVar('STAMP') + '.' + task + '.taint'
     bb.utils.mkdirhier(os.path.dirname(taintfn))
     # The specific content of the taint file is not really important,
     # we just need it to be random, so a random UUID is used
@@ -861,3 +862,46 @@
         if task in deps:
             deps.remove(task)
             d.setVarFlag(bbtask, 'deps', deps)
+
+def preceedtask(task, with_recrdeptasks, d):
+    """
+    Returns a set of tasks in the current recipe which were specified as
+    precondition by the task itself ("after") or which listed themselves
+    as precondition ("before"). Preceeding tasks specified via the
+    "recrdeptask" are included in the result only if requested. Beware
+    that this may lead to the task itself being listed.
+    """
+    preceed = set()
+    preceed.update(d.getVarFlag(task, 'deps') or [])
+    if with_recrdeptasks:
+        recrdeptask = d.getVarFlag(task, 'recrdeptask')
+        if recrdeptask:
+            preceed.update(recrdeptask.split())
+    return preceed
+
+def tasksbetween(task_start, task_end, d):
+    """
+    Return the list of tasks between two tasks in the current recipe,
+    where task_start is to start at and task_end is the task to end at
+    (and task_end has a dependency chain back to task_start).
+    """
+    outtasks = []
+    tasks = list(filter(lambda k: d.getVarFlag(k, "task"), d.keys()))
+    def follow_chain(task, endtask, chain=None):
+        if not chain:
+            chain = []
+        chain.append(task)
+        for othertask in tasks:
+            if othertask == task:
+                continue
+            if task == endtask:
+                for ctask in chain:
+                    if ctask not in outtasks:
+                        outtasks.append(ctask)
+            else:
+                deps = d.getVarFlag(othertask, 'deps', False)
+                if task in deps:
+                    follow_chain(othertask, endtask, chain)
+        chain.pop()
+    follow_chain(task_start, task_end)
+    return outtasks
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/cache.py b/import-layers/yocto-poky/bitbake/lib/bb/cache.py
index dd9cfdf..e7eeb4f 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/cache.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/cache.py
@@ -37,7 +37,7 @@
 
 logger = logging.getLogger("BitBake.Cache")
 
-__cache_version__ = "150"
+__cache_version__ = "151"
 
 def getCacheFile(path, filename, data_hash):
     return os.path.join(path, filename + "." + data_hash)
@@ -71,7 +71,7 @@
 
     @classmethod
     def flaglist(cls, flag, varlist, metadata, squash=False):
-        out_dict = dict((var, metadata.getVarFlag(var, flag, True))
+        out_dict = dict((var, metadata.getVarFlag(var, flag))
                     for var in varlist)
         if squash:
             return dict((k,v) for (k,v) in out_dict.items() if v)
@@ -296,7 +296,7 @@
     bb_data.setVar("__BBMULTICONFIG", mc)
 
     # expand tmpdir to include this topdir
-    bb_data.setVar('TMPDIR', bb_data.getVar('TMPDIR', True) or "")
+    bb_data.setVar('TMPDIR', bb_data.getVar('TMPDIR') or "")
     bbfile_loc = os.path.abspath(os.path.dirname(bbfile))
     oldpath = os.path.abspath(os.getcwd())
     bb.parse.cached_mtime_noerror(bbfile_loc)
@@ -378,7 +378,7 @@
         # It will be used later for deciding whether we 
         # need extra cache file dump/load support 
         self.caches_array = caches_array
-        self.cachedir = data.getVar("CACHE", True)
+        self.cachedir = data.getVar("CACHE")
         self.clean = set()
         self.checked = set()
         self.depends_cache = {}
@@ -462,6 +462,10 @@
                         self.depends_cache[key] = [value]
                     # only fire events on even percentage boundaries
                     current_progress = cachefile.tell() + previous_progress
+                    if current_progress > cachesize:
+                        # we might have calculated incorrect total size because a file
+                        # might've been written out just after we checked its size
+                        cachesize = current_progress
                     current_percent = 100 * current_progress / cachesize
                     if current_percent > previous_percent:
                         previous_percent = current_percent
@@ -792,8 +796,8 @@
         self.cachedata_extras = self.create_cachedata()
 
     def init_cache(self, d, cache_file_name=None):
-        cachedir = (d.getVar("PERSISTENT_DIR", True) or
-                    d.getVar("CACHE", True))
+        cachedir = (d.getVar("PERSISTENT_DIR") or
+                    d.getVar("CACHE"))
         if cachedir in [None, '']:
             return
         bb.utils.mkdirhier(cachedir)
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/codeparser.py b/import-layers/yocto-poky/bitbake/lib/bb/codeparser.py
index 5d2d440..530f44e 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/codeparser.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/codeparser.py
@@ -1,3 +1,22 @@
+"""
+BitBake code parser
+
+Parses actual code (i.e. python and shell) for functions and in-line
+expressions. Used mainly to determine dependencies on other functions
+and variables within the BitBake metadata. Also provides a cache for
+this information in order to speed up processing.
+
+(Not to be confused with the code that parses the metadata itself,
+see lib/bb/parse/ for that).
+
+NOTE: if you change how the parsers gather information you will almost
+certainly need to increment CodeParserCache.CACHE_VERSION below so that
+any existing codeparser cache gets invalidated. Additionally you'll need
+to increment __cache_version__ in cache.py in order to ensure that old
+recipe caches don't trigger "Taskhash mismatch" errors.
+
+"""
+
 import ast
 import sys
 import codegen
@@ -117,7 +136,11 @@
 
 class CodeParserCache(MultiProcessCache):
     cache_file_name = "bb_codeparser.dat"
-    CACHE_VERSION = 8
+    # NOTE: you must increment this if you change how the parsers gather information,
+    # so that an existing cache gets invalidated. Additionally you'll need
+    # to increment __cache_version__ in cache.py in order to ensure that old
+    # recipe caches don't trigger "Taskhash mismatch" errors.
+    CACHE_VERSION = 9
 
     def __init__(self):
         MultiProcessCache.__init__(self)
@@ -186,13 +209,15 @@
 
     def flush(self):
         for record in self.buffer:
-            self.target.handle(record)
+            if self.target.isEnabledFor(record.levelno):
+                self.target.handle(record)
         self.buffer = []
 
 class PythonParser():
     getvars = (".getVar", ".appendVar", ".prependVar")
     getvarflags = (".getVarFlag", ".appendVarFlag", ".prependVarFlag")
-    containsfuncs = ("bb.utils.contains", "base_contains", "bb.utils.contains_any")
+    containsfuncs = ("bb.utils.contains", "base_contains")
+    containsanyfuncs = ("bb.utils.contains_any",  "bb.utils.filter")
     execfuncs = ("bb.build.exec_func", "bb.build.exec_task")
 
     def warn(self, func, arg):
@@ -211,13 +236,17 @@
 
     def visit_Call(self, node):
         name = self.called_node_name(node.func)
-        if name and (name.endswith(self.getvars) or name.endswith(self.getvarflags) or name in self.containsfuncs):
+        if name and (name.endswith(self.getvars) or name.endswith(self.getvarflags) or name in self.containsfuncs or name in self.containsanyfuncs):
             if isinstance(node.args[0], ast.Str):
                 varname = node.args[0].s
                 if name in self.containsfuncs and isinstance(node.args[1], ast.Str):
                     if varname not in self.contains:
                         self.contains[varname] = set()
                     self.contains[varname].add(node.args[1].s)
+                elif name in self.containsanyfuncs and isinstance(node.args[1], ast.Str):
+                    if varname not in self.contains:
+                        self.contains[varname] = set()
+                    self.contains[varname].update(node.args[1].s.split())
                 elif name.endswith(self.getvarflags):
                     if isinstance(node.args[1], ast.Str):
                         self.references.add('%s[%s]' % (varname, node.args[1].s))
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/command.py b/import-layers/yocto-poky/bitbake/lib/bb/command.py
index caa3e4d..a919f58 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/command.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/command.py
@@ -28,8 +28,15 @@
 Commands are queued in a CommandQueue
 """
 
+from collections import OrderedDict, defaultdict
+
 import bb.event
 import bb.cooker
+import bb.remotedata
+
+class DataStoreConnectionHandle(object):
+    def __init__(self, dsindex=0):
+        self.dsindex = dsindex
 
 class CommandCompleted(bb.event.Event):
     pass
@@ -55,6 +62,7 @@
         self.cooker = cooker
         self.cmds_sync = CommandsSync()
         self.cmds_async = CommandsAsync()
+        self.remotedatastores = bb.remotedata.RemoteDatastores(cooker)
 
         # FIXME Add lock for this
         self.currentAsyncCommand = None
@@ -125,14 +133,20 @@
 
     def finishAsyncCommand(self, msg=None, code=None):
         if msg or msg == "":
-            bb.event.fire(CommandFailed(msg), self.cooker.expanded_data)
+            bb.event.fire(CommandFailed(msg), self.cooker.data)
         elif code:
-            bb.event.fire(CommandExit(code), self.cooker.expanded_data)
+            bb.event.fire(CommandExit(code), self.cooker.data)
         else:
-            bb.event.fire(CommandCompleted(), self.cooker.expanded_data)
+            bb.event.fire(CommandCompleted(), self.cooker.data)
         self.currentAsyncCommand = None
         self.cooker.finishcommand()
 
+def split_mc_pn(pn):
+    if pn.startswith("multiconfig:"):
+        _, mc, pn = pn.split(":", 2)
+        return (mc, pn)
+    return ('', pn)
+
 class CommandsSync:
     """
     A class of synchronous commands
@@ -179,6 +193,7 @@
         """
         varname = params[0]
         value = str(params[1])
+        command.cooker.extraconfigdata[varname] = value
         command.cooker.data.setVar(varname, value)
 
     def getSetVariable(self, command, params):
@@ -295,9 +310,274 @@
     def updateConfig(self, command, params):
         options = params[0]
         environment = params[1]
-        command.cooker.updateConfigOpts(options, environment)
+        cmdline = params[2]
+        command.cooker.updateConfigOpts(options, environment, cmdline)
     updateConfig.needconfig = False
 
+    def parseConfiguration(self, command, params):
+        """Instruct bitbake to parse its configuration
+        NOTE: it is only necessary to call this if you aren't calling any normal action
+        (otherwise parsing is taken care of automatically)
+        """
+        command.cooker.parseConfiguration()
+    parseConfiguration.needconfig = False
+
+    def getLayerPriorities(self, command, params):
+        ret = []
+        # regex objects cannot be marshalled by xmlrpc
+        for collection, pattern, regex, pri in command.cooker.bbfile_config_priorities:
+            ret.append((collection, pattern, regex.pattern, pri))
+        return ret
+    getLayerPriorities.readonly = True
+
+    def getRecipes(self, command, params):
+        try:
+            mc = params[0]
+        except IndexError:
+            mc = ''
+        return list(command.cooker.recipecaches[mc].pkg_pn.items())
+    getRecipes.readonly = True
+
+    def getRecipeDepends(self, command, params):
+        try:
+            mc = params[0]
+        except IndexError:
+            mc = ''
+        return list(command.cooker.recipecaches[mc].deps.items())
+    getRecipeDepends.readonly = True
+
+    def getRecipeVersions(self, command, params):
+        try:
+            mc = params[0]
+        except IndexError:
+            mc = ''
+        return command.cooker.recipecaches[mc].pkg_pepvpr
+    getRecipeVersions.readonly = True
+
+    def getRuntimeDepends(self, command, params):
+        ret = []
+        try:
+            mc = params[0]
+        except IndexError:
+            mc = ''
+        rundeps = command.cooker.recipecaches[mc].rundeps
+        for key, value in rundeps.items():
+            if isinstance(value, defaultdict):
+                value = dict(value)
+            ret.append((key, value))
+        return ret
+    getRuntimeDepends.readonly = True
+
+    def getRuntimeRecommends(self, command, params):
+        ret = []
+        try:
+            mc = params[0]
+        except IndexError:
+            mc = ''
+        runrecs = command.cooker.recipecaches[mc].runrecs
+        for key, value in runrecs.items():
+            if isinstance(value, defaultdict):
+                value = dict(value)
+            ret.append((key, value))
+        return ret
+    getRuntimeRecommends.readonly = True
+
+    def getRecipeInherits(self, command, params):
+        try:
+            mc = params[0]
+        except IndexError:
+            mc = ''
+        return command.cooker.recipecaches[mc].inherits
+    getRecipeInherits.readonly = True
+
+    def getBbFilePriority(self, command, params):
+        try:
+            mc = params[0]
+        except IndexError:
+            mc = ''
+        return command.cooker.recipecaches[mc].bbfile_priority
+    getBbFilePriority.readonly = True
+
+    def getDefaultPreference(self, command, params):
+        try:
+            mc = params[0]
+        except IndexError:
+            mc = ''
+        return command.cooker.recipecaches[mc].pkg_dp
+    getDefaultPreference.readonly = True
+
+    def getSkippedRecipes(self, command, params):
+        # Return list sorted by reverse priority order
+        import bb.cache
+        skipdict = OrderedDict(sorted(command.cooker.skiplist.items(),
+                                      key=lambda x: (-command.cooker.collection.calc_bbfile_priority(bb.cache.virtualfn2realfn(x[0])[0]), x[0])))
+        return list(skipdict.items())
+    getSkippedRecipes.readonly = True
+
+    def getOverlayedRecipes(self, command, params):
+        return list(command.cooker.collection.overlayed.items())
+    getOverlayedRecipes.readonly = True
+
+    def getFileAppends(self, command, params):
+        fn = params[0]
+        return command.cooker.collection.get_file_appends(fn)
+    getFileAppends.readonly = True
+
+    def getAllAppends(self, command, params):
+        return command.cooker.collection.bbappends
+    getAllAppends.readonly = True
+
+    def findProviders(self, command, params):
+        return command.cooker.findProviders()
+    findProviders.readonly = True
+
+    def findBestProvider(self, command, params):
+        (mc, pn) = split_mc_pn(params[0])
+        return command.cooker.findBestProvider(pn, mc)
+    findBestProvider.readonly = True
+
+    def allProviders(self, command, params):
+        try:
+            mc = params[0]
+        except IndexError:
+            mc = ''
+        return list(bb.providers.allProviders(command.cooker.recipecaches[mc]).items())
+    allProviders.readonly = True
+
+    def getRuntimeProviders(self, command, params):
+        rprovide = params[0]
+        try:
+            mc = params[1]
+        except IndexError:
+            mc = ''
+        all_p = bb.providers.getRuntimeProviders(command.cooker.recipecaches[mc], rprovide)
+        if all_p:
+            best = bb.providers.filterProvidersRunTime(all_p, rprovide,
+                            command.cooker.data,
+                            command.cooker.recipecaches[mc])[0][0]
+        else:
+            best = None
+        return all_p, best
+    getRuntimeProviders.readonly = True
+
+    def dataStoreConnectorFindVar(self, command, params):
+        dsindex = params[0]
+        name = params[1]
+        datastore = command.remotedatastores[dsindex]
+        value, overridedata = datastore._findVar(name)
+
+        if value:
+            content = value.get('_content', None)
+            if isinstance(content, bb.data_smart.DataSmart):
+                # Value is a datastore (e.g. BB_ORIGENV) - need to handle this carefully
+                idx = command.remotedatastores.check_store(content, True)
+                return {'_content': DataStoreConnectionHandle(idx),
+                        '_connector_origtype': 'DataStoreConnectionHandle',
+                        '_connector_overrides': overridedata}
+            elif isinstance(content, set):
+                return {'_content': list(content),
+                        '_connector_origtype': 'set',
+                        '_connector_overrides': overridedata}
+            else:
+                value['_connector_overrides'] = overridedata
+        else:
+            value = {}
+            value['_connector_overrides'] = overridedata
+        return value
+    dataStoreConnectorFindVar.readonly = True
+
+    def dataStoreConnectorGetKeys(self, command, params):
+        dsindex = params[0]
+        datastore = command.remotedatastores[dsindex]
+        return list(datastore.keys())
+    dataStoreConnectorGetKeys.readonly = True
+
+    def dataStoreConnectorGetVarHistory(self, command, params):
+        dsindex = params[0]
+        name = params[1]
+        datastore = command.remotedatastores[dsindex]
+        return datastore.varhistory.variable(name)
+    dataStoreConnectorGetVarHistory.readonly = True
+
+    def dataStoreConnectorExpandPythonRef(self, command, params):
+        config_data_dict = params[0]
+        varname = params[1]
+        expr = params[2]
+
+        config_data = command.remotedatastores.receive_datastore(config_data_dict)
+
+        varparse = bb.data_smart.VariableParse(varname, config_data)
+        return varparse.python_sub(expr)
+
+    def dataStoreConnectorRelease(self, command, params):
+        dsindex = params[0]
+        if dsindex <= 0:
+            raise CommandError('dataStoreConnectorRelease: invalid index %d' % dsindex)
+        command.remotedatastores.release(dsindex)
+
+    def dataStoreConnectorSetVarFlag(self, command, params):
+        dsindex = params[0]
+        name = params[1]
+        flag = params[2]
+        value = params[3]
+        datastore = command.remotedatastores[dsindex]
+        datastore.setVarFlag(name, flag, value)
+
+    def dataStoreConnectorDelVar(self, command, params):
+        dsindex = params[0]
+        name = params[1]
+        datastore = command.remotedatastores[dsindex]
+        if len(params) > 2:
+            flag = params[2]
+            datastore.delVarFlag(name, flag)
+        else:
+            datastore.delVar(name)
+
+    def dataStoreConnectorRenameVar(self, command, params):
+        dsindex = params[0]
+        name = params[1]
+        newname = params[2]
+        datastore = command.remotedatastores[dsindex]
+        datastore.renameVar(name, newname)
+
+    def parseRecipeFile(self, command, params):
+        """
+        Parse the specified recipe file (with or without bbappends)
+        and return a datastore object representing the environment
+        for the recipe.
+        """
+        fn = params[0]
+        appends = params[1]
+        appendlist = params[2]
+        if len(params) > 3:
+            config_data_dict = params[3]
+            config_data = command.remotedatastores.receive_datastore(config_data_dict)
+        else:
+            config_data = None
+
+        if appends:
+            if appendlist is not None:
+                appendfiles = appendlist
+            else:
+                appendfiles = command.cooker.collection.get_file_appends(fn)
+        else:
+            appendfiles = []
+        # We are calling bb.cache locally here rather than on the server,
+        # but that's OK because it doesn't actually need anything from
+        # the server barring the global datastore (which we have a remote
+        # version of)
+        if config_data:
+            # We have to use a different function here if we're passing in a datastore
+            # NOTE: we took a copy above, so we don't do it here again
+            envdata = bb.cache.parse_recipe(config_data, fn, appendfiles)['']
+        else:
+            # Use the standard path
+            parser = bb.cache.NoCache(command.cooker.databuilder)
+            envdata = parser.loadDataFull(fn, appendfiles)
+        idx = command.remotedatastores.store(envdata)
+        return DataStoreConnectionHandle(idx)
+    parseRecipeFile.readonly = True
+
 class CommandsAsync:
     """
     A class of asynchronous commands
@@ -311,8 +591,12 @@
         """
         bfile = params[0]
         task = params[1]
+        if len(params) > 2:
+            hidewarning = params[2]
+        else:
+            hidewarning = False
 
-        command.cooker.buildFile(bfile, task)
+        command.cooker.buildFile(bfile, task, hidewarning)
     buildFile.needcache = False
 
     def buildTargets(self, command, params):
@@ -472,3 +756,11 @@
         command.finishAsyncCommand()
     resetCooker.needcache = False
 
+    def clientComplete(self, command, params):
+        """
+        Do the right thing when the controlling client exits
+        """
+        command.cooker.clientComplete()
+        command.finishAsyncCommand()
+    clientComplete.needcache = False
+
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/cooker.py b/import-layers/yocto-poky/bitbake/lib/bb/cooker.py
index 07897be..3c9e88c 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/cooker.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/cooker.py
@@ -192,6 +192,8 @@
         bb.parse.__mtime_cache = {}
         bb.parse.BBHandler.cached_statements = {}
 
+        self.ui_cmdline = None
+
         self.initConfigurationData()
 
         # we log all events to a file if so directed
@@ -271,12 +273,15 @@
             self.inotify_modified_files.append(event.pathname)
         self.parsecache_valid = False
 
-    def add_filewatch(self, deps, watcher=None):
+    def add_filewatch(self, deps, watcher=None, dirs=False):
         if not watcher:
             watcher = self.watcher
         for i in deps:
             watcher.bbwatchedfiles.append(i[0])
-            f = os.path.dirname(i[0])
+            if dirs:
+                f = i[0]
+            else:
+                f = os.path.dirname(i[0])
             if f in watcher.bbseen:
                 continue
             watcher.bbseen.append(f)
@@ -331,7 +336,7 @@
         # Need to preserve BB_CONSOLELOG over resets
         consolelog = None
         if hasattr(self, "data"):
-            consolelog = self.data.getVar("BB_CONSOLELOG", True)
+            consolelog = self.data.getVar("BB_CONSOLELOG")
 
         if CookerFeatures.BASEDATASTORE_TRACKING in self.featureset:
             self.enableDataTracking()
@@ -358,17 +363,18 @@
         self.databuilder.parseBaseConfiguration()
         self.data = self.databuilder.data
         self.data_hash = self.databuilder.data_hash
+        self.extraconfigdata = {}
 
         if consolelog:
             self.data.setVar("BB_CONSOLELOG", consolelog)
 
+        self.data.setVar('BB_CMDLINE', self.ui_cmdline)
+
         #
         # Copy of the data store which has been expanded.
         # Used for firing events and accessing variables where expansion needs to be accounted for
         #
-        self.expanded_data = bb.data.createCopy(self.data)
-        bb.data.update_data(self.expanded_data)
-        bb.parse.init_parser(self.expanded_data)
+        bb.parse.init_parser(self.data)
 
         if CookerFeatures.BASEDATASTORE_TRACKING in self.featureset:
             self.disableDataTracking()
@@ -526,7 +532,7 @@
             bb.msg.loggerVerboseLogs = True
 
         # Change nice level if we're asked to
-        nice = self.data.getVar("BB_NICE_LEVEL", True)
+        nice = self.data.getVar("BB_NICE_LEVEL")
         if nice:
             curnice = os.nice(0)
             nice = int(nice) - curnice
@@ -539,9 +545,10 @@
         for mc in self.multiconfigs:
             self.recipecaches[mc] = bb.cache.CacheData(self.caches_array)
 
-        self.handleCollections(self.data.getVar("BBFILE_COLLECTIONS", True))
+        self.handleCollections(self.data.getVar("BBFILE_COLLECTIONS"))
 
-    def updateConfigOpts(self, options, environment):
+    def updateConfigOpts(self, options, environment, cmdline):
+        self.ui_cmdline = cmdline
         clean = True
         for o in options:
             if o in ['prefile', 'postfile']:
@@ -583,13 +590,12 @@
 
     def showVersions(self):
 
-        pkg_pn = self.recipecaches[''].pkg_pn
-        (latest_versions, preferred_versions) = bb.providers.findProviders(self.data, self.recipecaches[''], pkg_pn)
+        (latest_versions, preferred_versions) = self.findProviders()
 
         logger.plain("%-35s %25s %25s", "Recipe Name", "Latest Version", "Preferred Version")
         logger.plain("%-35s %25s %25s\n", "===========", "==============", "=================")
 
-        for p in sorted(pkg_pn):
+        for p in sorted(self.recipecaches[''].pkg_pn):
             pref = preferred_versions[p]
             latest = latest_versions[p]
 
@@ -619,7 +625,7 @@
             fn = self.matchFile(fn)
             fn = bb.cache.realfn2virtual(fn, cls, mc)
         elif len(pkgs_to_build) == 1:
-            ignore = self.expanded_data.getVar("ASSUME_PROVIDED", True) or ""
+            ignore = self.data.getVar("ASSUME_PROVIDED") or ""
             if pkgs_to_build[0] in set(ignore.split()):
                 bb.fatal("%s is in ASSUME_PROVIDED" % pkgs_to_build[0])
 
@@ -644,14 +650,13 @@
             logger.plain(env.getvalue())
 
         # emit variables and shell functions
-        data.update_data(envdata)
         with closing(StringIO()) as env:
             data.emit_env(env, envdata, True)
             logger.plain(env.getvalue())
 
         # emit the metadata which isnt valid shell
         data.expandKeys(envdata)
-        for e in envdata.keys():
+        for e in sorted(envdata.keys()):
             if envdata.getVarFlag(e, 'func', False) and envdata.getVarFlag(e, 'python', False):
                 logger.plain("\npython %s () {\n%s}\n", e, envdata.getVar(e, False))
 
@@ -705,7 +710,6 @@
         for mc in self.multiconfigs:
             taskdata[mc] = bb.taskdata.TaskData(abort, skiplist=self.skiplist, allowincomplete=allowincomplete)
             localdata[mc] = data.createCopy(self.databuilder.mcdata[mc])
-            bb.data.update_data(localdata[mc])
             bb.data.expandKeys(localdata[mc])
 
         current = 0
@@ -766,7 +770,7 @@
     @staticmethod
     def add_mc_prefix(mc, pn):
         if mc:
-            return "multiconfig:%s.%s" % (mc, pn)
+            return "multiconfig:%s:%s" % (mc, pn)
         return pn
 
     def buildDependTree(self, rq, taskdata):
@@ -951,62 +955,54 @@
 
         depgraph = self.generateTaskDepTreeData(pkgs_to_build, task)
 
-        # Prints a flattened form of package-depends below where subpackages of a package are merged into the main pn
-        depends_file = open('pn-depends.dot', 'w' )
-        buildlist_file = open('pn-buildlist', 'w' )
-        print("digraph depends {", file=depends_file)
-        for pn in depgraph["pn"]:
-            fn = depgraph["pn"][pn]["filename"]
-            version = depgraph["pn"][pn]["version"]
-            print('"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn), file=depends_file)
-            print("%s" % pn, file=buildlist_file)
-        buildlist_file.close()
+        with open('pn-buildlist', 'w') as f:
+            for pn in depgraph["pn"]:
+                f.write(pn + "\n")
         logger.info("PN build list saved to 'pn-buildlist'")
-        for pn in depgraph["depends"]:
-            for depend in depgraph["depends"][pn]:
-                print('"%s" -> "%s" [style=solid]' % (pn, depend), file=depends_file)
-        for pn in depgraph["rdepends-pn"]:
-            for rdepend in depgraph["rdepends-pn"][pn]:
-                print('"%s" -> "%s" [style=dashed]' % (pn, rdepend), file=depends_file)
-        print("}", file=depends_file)
-        depends_file.close()
-        logger.info("PN dependencies saved to 'pn-depends.dot'")
 
-        depends_file = open('package-depends.dot', 'w' )
-        print("digraph depends {", file=depends_file)
-        for package in depgraph["packages"]:
-            pn = depgraph["packages"][package]["pn"]
-            fn = depgraph["packages"][package]["filename"]
-            version = depgraph["packages"][package]["version"]
-            if package == pn:
-                print('"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn), file=depends_file)
-            else:
-                print('"%s" [label="%s(%s) %s\\n%s"]' % (package, package, pn, version, fn), file=depends_file)
-            for depend in depgraph["depends"][pn]:
-                print('"%s" -> "%s" [style=solid]' % (package, depend), file=depends_file)
-        for package in depgraph["rdepends-pkg"]:
-            for rdepend in depgraph["rdepends-pkg"][package]:
-                print('"%s" -> "%s" [style=dashed]' % (package, rdepend), file=depends_file)
-        for package in depgraph["rrecs-pkg"]:
-            for rdepend in depgraph["rrecs-pkg"][package]:
-                print('"%s" -> "%s" [style=dotted]' % (package, rdepend), file=depends_file)
-        print("}", file=depends_file)
-        depends_file.close()
-        logger.info("Package dependencies saved to 'package-depends.dot'")
+        # Remove old format output files to ensure no confusion with stale data
+        try:
+            os.unlink('pn-depends.dot')
+        except FileNotFoundError:
+            pass
+        try:
+            os.unlink('package-depends.dot')
+        except FileNotFoundError:
+            pass
 
-        tdepends_file = open('task-depends.dot', 'w' )
-        print("digraph depends {", file=tdepends_file)
-        for task in depgraph["tdepends"]:
-            (pn, taskname) = task.rsplit(".", 1)
-            fn = depgraph["pn"][pn]["filename"]
-            version = depgraph["pn"][pn]["version"]
-            print('"%s.%s" [label="%s %s\\n%s\\n%s"]' % (pn, taskname, pn, taskname, version, fn), file=tdepends_file)
-            for dep in depgraph["tdepends"][task]:
-                print('"%s" -> "%s"' % (task, dep), file=tdepends_file)
-        print("}", file=tdepends_file)
-        tdepends_file.close()
+        with open('task-depends.dot', 'w') as f:
+            f.write("digraph depends {\n")
+            for task in depgraph["tdepends"]:
+                (pn, taskname) = task.rsplit(".", 1)
+                fn = depgraph["pn"][pn]["filename"]
+                version = depgraph["pn"][pn]["version"]
+                f.write('"%s.%s" [label="%s %s\\n%s\\n%s"]\n' % (pn, taskname, pn, taskname, version, fn))
+                for dep in depgraph["tdepends"][task]:
+                    f.write('"%s" -> "%s"\n' % (task, dep))
+            f.write("}\n")
         logger.info("Task dependencies saved to 'task-depends.dot'")
 
+        with open('recipe-depends.dot', 'w') as f:
+            f.write("digraph depends {\n")
+            pndeps = {}
+            for task in depgraph["tdepends"]:
+                (pn, taskname) = task.rsplit(".", 1)
+                if pn not in pndeps:
+                    pndeps[pn] = set()
+                for dep in depgraph["tdepends"][task]:
+                    (deppn, deptaskname) = dep.rsplit(".", 1)
+                    pndeps[pn].add(deppn)
+            for pn in pndeps:
+                fn = depgraph["pn"][pn]["filename"]
+                version = depgraph["pn"][pn]["version"]
+                f.write('"%s" [label="%s\\n%s\\n%s"]\n' % (pn, pn, version, fn))
+                for dep in pndeps[pn]:
+                    if dep == pn:
+                        continue
+                    f.write('"%s" -> "%s"\n' % (pn, dep))
+            f.write("}\n")
+        logger.info("Flatened recipe dependencies saved to 'recipe-depends.dot'")
+
     def show_appends_with_no_recipes(self):
         # Determine which bbappends haven't been applied
 
@@ -1037,11 +1033,10 @@
 
         for mc in self.multiconfigs:
             localdata = data.createCopy(self.databuilder.mcdata[mc])
-            bb.data.update_data(localdata)
             bb.data.expandKeys(localdata)
 
             # Handle PREFERRED_PROVIDERS
-            for p in (localdata.getVar('PREFERRED_PROVIDERS', True) or "").split():
+            for p in (localdata.getVar('PREFERRED_PROVIDERS') or "").split():
                 try:
                     (providee, provider) = p.split(':')
                 except:
@@ -1052,7 +1047,7 @@
                 self.recipecaches[mc].preferred[providee] = provider
 
     def findCoreBaseFiles(self, subdir, configfile):
-        corebase = self.data.getVar('COREBASE', True) or ""
+        corebase = self.data.getVar('COREBASE') or ""
         paths = []
         for root, dirs, files in os.walk(corebase + '/' + subdir):
             for d in dirs:
@@ -1102,7 +1097,7 @@
         """
 
         matches = []
-        bbpaths = self.data.getVar('BBPATH', True).split(':')
+        bbpaths = self.data.getVar('BBPATH').split(':')
         for path in bbpaths:
             dirpath = os.path.join(path, directory)
             if os.path.exists(dirpath):
@@ -1114,6 +1109,20 @@
         if matches:
             bb.event.fire(bb.event.FilesMatchingFound(filepattern, matches), self.data)
 
+    def findProviders(self, mc=''):
+        return bb.providers.findProviders(self.data, self.recipecaches[mc], self.recipecaches[mc].pkg_pn)
+
+    def findBestProvider(self, pn, mc=''):
+        if pn in self.recipecaches[mc].providers:
+            filenames = self.recipecaches[mc].providers[pn]
+            eligible, foundUnique = bb.providers.filterProviders(filenames, pn, self.data, self.recipecaches[mc])
+            filename = eligible[0]
+            return None, None, None, filename
+        elif pn in self.recipecaches[mc].pkg_pn:
+            return bb.providers.findBestProvider(pn, self.data, self.recipecaches[mc], self.recipecaches[mc].pkg_pn)
+        else:
+            return None, None, None, None
+
     def findConfigFiles(self, varname):
         """
         Find config files which are appropriate values for varname.
@@ -1124,7 +1133,7 @@
 
         data = self.data
         # iterate configs
-        bbpaths = data.getVar('BBPATH', True).split(':')
+        bbpaths = data.getVar('BBPATH').split(':')
         for path in bbpaths:
             confpath = os.path.join(path, "conf", var)
             if os.path.exists(confpath):
@@ -1193,7 +1202,7 @@
                 bb.debug(1,'Processing %s in collection list' % (c))
 
                 # Get collection priority if defined explicitly
-                priority = self.data.getVar("BBFILE_PRIORITY_%s" % c, True)
+                priority = self.data.getVar("BBFILE_PRIORITY_%s" % c)
                 if priority:
                     try:
                         prio = int(priority)
@@ -1207,7 +1216,7 @@
                     collection_priorities[c] = None
 
                 # Check dependencies and store information for priority calculation
-                deps = self.data.getVar("LAYERDEPENDS_%s" % c, True)
+                deps = self.data.getVar("LAYERDEPENDS_%s" % c)
                 if deps:
                     try:
                         depDict = bb.utils.explode_dep_versions2(deps)
@@ -1216,7 +1225,7 @@
                     for dep, oplist in list(depDict.items()):
                         if dep in collection_list:
                             for opstr in oplist:
-                                layerver = self.data.getVar("LAYERVERSION_%s" % dep, True)
+                                layerver = self.data.getVar("LAYERVERSION_%s" % dep)
                                 (op, depver) = opstr.split()
                                 if layerver:
                                     try:
@@ -1237,7 +1246,7 @@
                     collection_depends[c] = []
 
                 # Check recommends and store information for priority calculation
-                recs = self.data.getVar("LAYERRECOMMENDS_%s" % c, True)
+                recs = self.data.getVar("LAYERRECOMMENDS_%s" % c)
                 if recs:
                     try:
                         recDict = bb.utils.explode_dep_versions2(recs)
@@ -1247,7 +1256,7 @@
                         if rec in collection_list:
                             if oplist:
                                 opstr = oplist[0]
-                                layerver = self.data.getVar("LAYERVERSION_%s" % rec, True)
+                                layerver = self.data.getVar("LAYERVERSION_%s" % rec)
                                 if layerver:
                                     (op, recver) = opstr.split()
                                     try:
@@ -1281,17 +1290,21 @@
             # Calculate all layer priorities using calc_layer_priority and store in bbfile_config_priorities
             for c in collection_list:
                 calc_layer_priority(c)
-                regex = self.data.getVar("BBFILE_PATTERN_%s" % c, True)
+                regex = self.data.getVar("BBFILE_PATTERN_%s" % c)
                 if regex == None:
                     parselog.error("BBFILE_PATTERN_%s not defined" % c)
                     errors = True
                     continue
-                try:
-                    cre = re.compile(regex)
-                except re.error:
-                    parselog.error("BBFILE_PATTERN_%s \"%s\" is not a valid regular expression", c, regex)
-                    errors = True
-                    continue
+                elif regex == "":
+                    parselog.debug(1, "BBFILE_PATTERN_%s is empty" % c)
+                    errors = False
+                else:
+                    try:
+                        cre = re.compile(regex)
+                    except re.error:
+                        parselog.error("BBFILE_PATTERN_%s \"%s\" is not a valid regular expression", c, regex)
+                        errors = True
+                        continue
                 self.bbfile_config_priorities.append((c, regex, cre, collection_priorities[c]))
         if errors:
             # We've already printed the actual error(s)
@@ -1316,7 +1329,7 @@
             bf = os.path.abspath(bf)
 
         self.collection = CookerCollectFiles(self.bbfile_config_priorities)
-        filelist, masked = self.collection.collect_bbfiles(self.data, self.expanded_data)
+        filelist, masked, searchdirs = self.collection.collect_bbfiles(self.data, self.data)
         try:
             os.stat(bf)
             bf = os.path.abspath(bf)
@@ -1347,15 +1360,16 @@
             raise NoSpecificMatch
         return matches[0]
 
-    def buildFile(self, buildfile, task):
+    def buildFile(self, buildfile, task, hidewarning=False):
         """
         Build the file matching regexp buildfile
         """
-        bb.event.fire(bb.event.BuildInit(), self.expanded_data)
+        bb.event.fire(bb.event.BuildInit(), self.data)
 
-        # Too many people use -b because they think it's how you normally
-        # specify a target to be built, so show a warning
-        bb.warn("Buildfile specified, dependencies will not be handled. If this is not what you want, do not use -b / --buildfile.")
+        if not hidewarning:
+            # Too many people use -b because they think it's how you normally
+            # specify a target to be built, so show a warning
+            bb.warn("Buildfile specified, dependencies will not be handled. If this is not what you want, do not use -b / --buildfile.")
 
         # Parse the configuration here. We need to do it explicitly here since
         # buildFile() doesn't use the cache
@@ -1392,6 +1406,7 @@
         item = info_array[0].pn
         self.recipecaches[mc].ignored_dependencies = set()
         self.recipecaches[mc].bbfile_priority[fn] = 1
+        self.configuration.limited_deps = True
 
         # Remove external dependencies
         self.recipecaches[mc].task_deps[fn]['depends'] = {}
@@ -1409,8 +1424,8 @@
         taskdata[mc] = bb.taskdata.TaskData(self.configuration.abort)
         taskdata[mc].add_provider(self.data, self.recipecaches[mc], item)
 
-        buildname = self.data.getVar("BUILDNAME", True)
-        bb.event.fire(bb.event.BuildStarted(buildname, [item]), self.expanded_data)
+        buildname = self.data.getVar("BUILDNAME")
+        bb.event.fire(bb.event.BuildStarted(buildname, [item]), self.data)
 
         # Execute the runqueue
         runlist = [[mc, item, task, fn]]
@@ -1440,7 +1455,7 @@
                 return False
 
             if not retval:
-                bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runtaskentries), buildname, item, failures, interrupted), self.expanded_data)
+                bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runtaskentries), buildname, item, failures, interrupted), self.data)
                 self.command.finishAsyncCommand(msg)
                 return False
             if retval is True:
@@ -1495,7 +1510,7 @@
 
         packages = [target if ':' in target else '%s:%s' % (target, task) for target in targets]
 
-        bb.event.fire(bb.event.BuildInit(packages), self.expanded_data)
+        bb.event.fire(bb.event.BuildInit(packages), self.data)
 
         taskdata, runlist = self.buildTaskData(targets, task, self.configuration.abort)
 
@@ -1528,7 +1543,7 @@
                 v = self.data.getVar(k, expand)
                 if not k.startswith("__") and not isinstance(v, bb.data_smart.DataSmart):
                     dump[k] = {
-    'v' : v ,
+    'v' : str(v) ,
     'history' : self.data.varhistory.variable(k),
                     }
                     for d in flaglist:
@@ -1627,14 +1642,18 @@
                     bb.event.fire(bb.event.SanityCheck(False), self.databuilder.mcdata[mc])
 
             for mc in self.multiconfigs:
-                ignore = self.databuilder.mcdata[mc].getVar("ASSUME_PROVIDED", True) or ""
+                ignore = self.databuilder.mcdata[mc].getVar("ASSUME_PROVIDED") or ""
                 self.recipecaches[mc].ignored_dependencies = set(ignore.split())
 
                 for dep in self.configuration.extra_assume_provided:
                     self.recipecaches[mc].ignored_dependencies.add(dep)
 
             self.collection = CookerCollectFiles(self.bbfile_config_priorities)
-            (filelist, masked) = self.collection.collect_bbfiles(self.data, self.expanded_data)
+            (filelist, masked, searchdirs) = self.collection.collect_bbfiles(self.data, self.data)
+
+            # Add inotify watches for directories searched for bb/bbappend files
+            for dirent in searchdirs:
+                self.add_filewatch([[dirent]], dirs=True)
 
             self.parser = CookerParser(self, filelist, masked)
             self.parsecache_valid = True
@@ -1668,7 +1687,7 @@
         if len(pkgs_to_build) == 0:
             raise NothingToBuild
 
-        ignore = (self.expanded_data.getVar("ASSUME_PROVIDED", True) or "").split()
+        ignore = (self.data.getVar("ASSUME_PROVIDED") or "").split()
         for pkg in pkgs_to_build:
             if pkg in ignore:
                 parselog.warning("Explicit target \"%s\" is in ASSUME_PROVIDED, ignoring" % pkg)
@@ -1688,6 +1707,15 @@
             pkgs_to_build.remove('universe')
             for mc in self.multiconfigs:
                 for t in self.recipecaches[mc].universe_target:
+                    if task:
+                        foundtask = False
+                        for provider_fn in self.recipecaches[mc].providers[t]:
+                            if task in self.recipecaches[mc].task_deps[provider_fn]['tasks']:
+                                foundtask = True
+                                break
+                        if not foundtask:
+                            bb.debug(1, "Skipping %s for universe tasks as task %s doesn't exist" % (t, task))
+                            continue
                     if mc:
                         t = "multiconfig:" + mc + ":" + t
                     pkgs_to_build.append(t)
@@ -1701,13 +1729,13 @@
         try:
             self.prhost = prserv.serv.auto_start(self.data)
         except prserv.serv.PRServiceConfigError:
-            bb.event.fire(CookerExit(), self.expanded_data)
+            bb.event.fire(CookerExit(), self.data)
             self.state = state.error
         return
 
     def post_serve(self):
         prserv.serv.auto_shutdown(self.data)
-        bb.event.fire(CookerExit(), self.expanded_data)
+        bb.event.fire(CookerExit(), self.data)
         lockfile = self.lock.name
         self.lock.close()
         self.lock = None
@@ -1745,6 +1773,8 @@
 
         if self.parser:
             self.parser.shutdown(clean=not force, force=force)
+        self.notifier.stop()
+        self.confignotifier.stop()
 
     def finishcommand(self):
         self.state = state.initial
@@ -1752,6 +1782,13 @@
     def reset(self):
         self.initConfigurationData()
 
+    def clientComplete(self):
+        """Called when the client is done using the server"""
+        if self.configuration.server_only:
+            self.finishcommand()
+        else:
+            self.shutdown(True)
+
     def lockBitbake(self):
         if not hasattr(self, 'lock'):
             self.lock = None
@@ -1838,7 +1875,7 @@
 
         collectlog.debug(1, "collecting .bb files")
 
-        files = (config.getVar( "BBFILES", True) or "").split()
+        files = (config.getVar( "BBFILES") or "").split()
         config.setVar("BBFILES", " ".join(files))
 
         # Sort files by priority
@@ -1851,30 +1888,49 @@
             collectlog.error("no recipe files to build, check your BBPATH and BBFILES?")
             bb.event.fire(CookerExit(), eventdata)
 
-        # Can't use set here as order is important
-        newfiles = []
-        for f in files:
-            if os.path.isdir(f):
-                dirfiles = self.find_bbfiles(f)
-                for g in dirfiles:
-                    if g not in newfiles:
-                        newfiles.append(g)
-            else:
-                globbed = glob.glob(f)
-                if not globbed and os.path.exists(f):
-                    globbed = [f]
-                # glob gives files in order on disk. Sort to be deterministic.
-                for g in sorted(globbed):
-                    if g not in newfiles:
-                        newfiles.append(g)
+        # We need to track where we look so that we can add inotify watches. There
+        # is no nice way to do this, this is horrid. We intercept the os.listdir()
+        # calls while we run glob().
+        origlistdir = os.listdir
+        searchdirs = []
 
-        bbmask = config.getVar('BBMASK', True)
+        def ourlistdir(d):
+            searchdirs.append(d)
+            return origlistdir(d)
+
+        os.listdir = ourlistdir
+        try:
+            # Can't use set here as order is important
+            newfiles = []
+            for f in files:
+                if os.path.isdir(f):
+                    dirfiles = self.find_bbfiles(f)
+                    for g in dirfiles:
+                        if g not in newfiles:
+                            newfiles.append(g)
+                else:
+                    globbed = glob.glob(f)
+                    if not globbed and os.path.exists(f):
+                        globbed = [f]
+                    # glob gives files in order on disk. Sort to be deterministic.
+                    for g in sorted(globbed):
+                        if g not in newfiles:
+                            newfiles.append(g)
+        finally:
+            os.listdir = origlistdir
+
+        bbmask = config.getVar('BBMASK')
 
         if bbmask:
             # First validate the individual regular expressions and ignore any
             # that do not compile
             bbmasks = []
             for mask in bbmask.split():
+                # When constructing an older style single regex, it's possible for BBMASK
+                # to end up beginning with '|', which matches and masks _everything_.
+                if mask.startswith("|"):
+                    collectlog.warn("BBMASK contains regular expression beginning with '|', fixing: %s" % mask)
+                    mask = mask[1:]
                 try:
                     re.compile(mask)
                     bbmasks.append(mask)
@@ -1921,7 +1977,7 @@
                 topfile = bbfile_seen[base]
                 self.overlayed[topfile].append(f)
 
-        return (bbfiles, masked)
+        return (bbfiles, masked, searchdirs)
 
     def get_file_appends(self, fn):
         """
@@ -1964,7 +2020,7 @@
 
         for collection, pattern, regex, _ in self.bbfile_config_priorities:
             if regex in unmatched:
-                if d.getVar('BBFILE_PATTERN_IGNORE_EMPTY_%s' % collection, True) != '1':
+                if d.getVar('BBFILE_PATTERN_IGNORE_EMPTY_%s' % collection) != '1':
                     collectlog.warning("No bb files matched BBFILE_PATTERN_%s '%s'" % (collection, pattern))
 
         return priorities
@@ -2121,7 +2177,7 @@
         self.toparse = self.total - len(self.fromcache)
         self.progress_chunk = int(max(self.toparse / 100, 1))
 
-        self.num_processes = min(int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS", True) or
+        self.num_processes = min(int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS") or
                                  multiprocessing.cpu_count()), len(self.willparse))
 
         self.start()
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/cookerdata.py b/import-layers/yocto-poky/bitbake/lib/bb/cookerdata.py
index 98f56ac..e408a35 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/cookerdata.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/cookerdata.py
@@ -79,7 +79,7 @@
                   "prefile", "postfile"]:
             options[o] = getattr(self.options, o)
 
-        ret, error = server.runCommand(["updateConfig", options, environment])
+        ret, error = server.runCommand(["updateConfig", options, environment, sys.argv])
         if error:
                 raise Exception("Unable to update the server configuration with local parameters: %s" % error)
 
@@ -146,6 +146,9 @@
         self.tracking = False
         self.interface = []
         self.writeeventlog = False
+        self.server_only = False
+        self.limited_deps = False
+        self.runall = None
 
         self.env = {}
 
@@ -212,7 +215,7 @@
 
 def findConfigFile(configfile, data):
     search = []
-    bbpath = data.getVar("BBPATH", True)
+    bbpath = data.getVar("BBPATH")
     if bbpath:
         for i in bbpath.split(":"):
             search.append(os.path.join(i, "conf", configfile))
@@ -286,7 +289,7 @@
             self.data_hash = self.data.get_hash()
             self.mcdata[''] = self.data
 
-            multiconfig = (self.data.getVar("BBMULTICONFIG", True) or "").split()
+            multiconfig = (self.data.getVar("BBMULTICONFIG") or "").split()
             for config in multiconfig:
                 mcdata = self.parseConfigurationFiles(self.prefiles, self.postfiles, config)
                 bb.event.fire(bb.event.ConfigParsed(), mcdata)
@@ -320,7 +323,7 @@
             data.setVar("TOPDIR", os.path.dirname(os.path.dirname(layerconf)))
             data = parse_config_file(layerconf, data)
 
-            layers = (data.getVar('BBLAYERS', True) or "").split()
+            layers = (data.getVar('BBLAYERS') or "").split()
 
             data = bb.data.createCopy(data)
             approved = bb.utils.approved_variables()
@@ -343,7 +346,7 @@
             data.delVar('LAYERDIR_RE')
             data.delVar('LAYERDIR')
 
-        if not data.getVar("BBPATH", True):
+        if not data.getVar("BBPATH"):
             msg = "The BBPATH variable is not set"
             if not layerconf:
                 msg += (" and bitbake did not find a conf/bblayers.conf file in"
@@ -358,7 +361,7 @@
             data = parse_config_file(p, data)
 
         # Handle any INHERITs and inherit the base class
-        bbclasses  = ["base"] + (data.getVar('INHERIT', True) or "").split()
+        bbclasses  = ["base"] + (data.getVar('INHERIT') or "").split()
         for bbclass in bbclasses:
             data = _inherit(bbclass, data)
 
@@ -370,7 +373,7 @@
                 parselog.critical("Undefined event handler function '%s'" % var)
                 sys.exit(1)
             handlerln = int(data.getVarFlag(var, "lineno", False))
-            bb.event.register(var, data.getVar(var, False),  (data.getVarFlag(var, "eventmask", True) or "").split(), handlerfn, handlerln)
+            bb.event.register(var, data.getVar(var, False),  (data.getVarFlag(var, "eventmask") or "").split(), handlerfn, handlerln)
 
         data.setVar('BBINCLUDED',bb.parse.get_file_depends(data))
 
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/data.py b/import-layers/yocto-poky/bitbake/lib/bb/data.py
index c56965c..134afaa 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/data.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/data.py
@@ -78,59 +78,6 @@
     """Non-destructive var init for data structure"""
     d.initVar(var)
 
-
-def setVar(var, value, d):
-    """Set a variable to a given value"""
-    d.setVar(var, value)
-
-
-def getVar(var, d, exp = False):
-    """Gets the value of a variable"""
-    return d.getVar(var, exp)
-
-
-def renameVar(key, newkey, d):
-    """Renames a variable from key to newkey"""
-    d.renameVar(key, newkey)
-
-def delVar(var, d):
-    """Removes a variable from the data set"""
-    d.delVar(var)
-
-def appendVar(var, value, d):
-    """Append additional value to a variable"""
-    d.appendVar(var, value)
-
-def setVarFlag(var, flag, flagvalue, d):
-    """Set a flag for a given variable to a given value"""
-    d.setVarFlag(var, flag, flagvalue)
-
-def getVarFlag(var, flag, d):
-    """Gets given flag from given var"""
-    return d.getVarFlag(var, flag, False)
-
-def delVarFlag(var, flag, d):
-    """Removes a given flag from the variable's flags"""
-    d.delVarFlag(var, flag)
-
-def setVarFlags(var, flags, d):
-    """Set the flags for a given variable
-
-    Note:
-        setVarFlags will not clear previous
-        flags. Think of this method as
-        addVarFlags
-    """
-    d.setVarFlags(var, flags)
-
-def getVarFlags(var, d):
-    """Gets a variable's flags"""
-    return d.getVarFlags(var)
-
-def delVarFlags(var, d):
-    """Removes a variable's flags"""
-    d.delVarFlags(var)
-
 def keys(d):
     """Return a list of keys in d"""
     return d.keys()
@@ -174,7 +121,7 @@
     for s in savedenv.keys():
         if s in permitted:
             try:
-                d.setVar(s, savedenv.getVar(s, True), op = 'from env')
+                d.setVar(s, savedenv.getVar(s), op = 'from env')
                 if s in exportlist:
                     d.setVarFlag(s, "export", True, op = 'auto env export')
             except TypeError:
@@ -194,7 +141,7 @@
     try:
         if all:
             oval = d.getVar(var, False)
-        val = d.getVar(var, True)
+        val = d.getVar(var)
     except (KeyboardInterrupt, bb.build.FuncFailed):
         raise
     except Exception as exc:
@@ -249,7 +196,7 @@
     keys = sorted((key for key in d.keys() if not key.startswith("__")), key=isfunc)
     grouped = groupby(keys, isfunc)
     for isfunc, keys in grouped:
-        for key in keys:
+        for key in sorted(keys):
             emit_var(key, o, d, all and not isfunc) and o.write('\n')
 
 def exported_keys(d):
@@ -261,9 +208,9 @@
     k = list(exported_keys(d))
     for key in k:
         try:
-            value = d.getVar(key, True)
+            value = d.getVar(key)
         except Exception as err:
-            bb.warn("%s: Unable to export ${%s}: %s" % (d.getVar("FILE", True), key, err))
+            bb.warn("%s: Unable to export ${%s}: %s" % (d.getVar("FILE"), key, err))
             continue
 
         if value is not None:
@@ -273,13 +220,13 @@
     """Emits all items in the data store in a format such that it can be sourced by a shell."""
 
     keys = (key for key in d.keys() if not key.startswith("__") and not d.getVarFlag(key, "func", False))
-    for key in keys:
+    for key in sorted(keys):
         emit_var(key, o, d, False)
 
     o.write('\n')
     emit_var(func, o, d, False) and o.write('\n')
-    newdeps = bb.codeparser.ShellParser(func, logger).parse_shell(d.getVar(func, True))
-    newdeps |= set((d.getVarFlag(func, "vardeps", True) or "").split())
+    newdeps = bb.codeparser.ShellParser(func, logger).parse_shell(d.getVar(func))
+    newdeps |= set((d.getVarFlag(func, "vardeps") or "").split())
     seen = set()
     while newdeps:
         deps = newdeps
@@ -288,8 +235,8 @@
         for dep in deps:
             if d.getVarFlag(dep, "func", False) and not d.getVarFlag(dep, "python", False):
                emit_var(dep, o, d, False) and o.write('\n')
-               newdeps |=  bb.codeparser.ShellParser(dep, logger).parse_shell(d.getVar(dep, True))
-               newdeps |= set((d.getVarFlag(dep, "vardeps", True) or "").split())
+               newdeps |=  bb.codeparser.ShellParser(dep, logger).parse_shell(d.getVar(dep))
+               newdeps |= set((d.getVarFlag(dep, "vardeps") or "").split())
         newdeps -= seen
 
 _functionfmt = """
@@ -312,7 +259,7 @@
     pp = bb.codeparser.PythonParser(func, logger)
     pp.parse_python(d.getVar(func, False))
     newdeps = pp.execs
-    newdeps |= set((d.getVarFlag(func, "vardeps", True) or "").split())
+    newdeps |= set((d.getVarFlag(func, "vardeps") or "").split())
     seen = set()
     while newdeps:
         deps = newdeps
@@ -324,7 +271,7 @@
                pp = bb.codeparser.PythonParser(dep, logger)
                pp.parse_python(d.getVar(dep, False))
                newdeps |= pp.execs
-               newdeps |= set((d.getVarFlag(dep, "vardeps", True) or "").split())
+               newdeps |= set((d.getVarFlag(dep, "vardeps") or "").split())
         newdeps -= seen
 
 def update_data(d):
@@ -348,12 +295,14 @@
         def handle_contains(value, contains, d):
             newvalue = ""
             for k in sorted(contains):
-                l = (d.getVar(k, True) or "").split()
-                for word in sorted(contains[k]):
-                    if word in l:
-                        newvalue += "\n%s{%s} = Set" %  (k, word)
+                l = (d.getVar(k) or "").split()
+                for item in sorted(contains[k]):
+                    for word in item.split():
+                        if not word in l:
+                            newvalue += "\n%s{%s} = Unset" % (k, item)
+                            break
                     else:
-                        newvalue += "\n%s{%s} = Unset" %  (k, word)
+                        newvalue += "\n%s{%s} = Set" % (k, item)
             if not newvalue:
                 return value
             if not value:
@@ -366,7 +315,7 @@
             if varflags.get("python"):
                 parser = bb.codeparser.PythonParser(key, logger)
                 if value and "\t" in value:
-                    logger.warning("Variable %s contains tabs, please remove these (%s)" % (key, d.getVar("FILE", True)))
+                    logger.warning("Variable %s contains tabs, please remove these (%s)" % (key, d.getVar("FILE")))
                 parser.parse_python(value, filename=varflags.get("filename"), lineno=varflags.get("lineno"))
                 deps = deps | parser.references
                 deps = deps | (keys & parser.execs)
@@ -410,6 +359,8 @@
 
         deps |= set((vardeps or "").split())
         deps -= set(varflags.get("vardepsexclude", "").split())
+    except bb.parse.SkipRecipe:
+        raise
     except Exception as e:
         bb.warn("Exception during build_dependencies for %s" % key)
         raise
@@ -421,7 +372,7 @@
 
     keys = set(key for key in d if not key.startswith("__"))
     shelldeps = set(key for key in d.getVar("__exportlist", False) if d.getVarFlag(key, "export", False) and not d.getVarFlag(key, "unexport", False))
-    varflagsexcl = d.getVar('BB_SIGNATURE_EXCLUDE_FLAGS', True)
+    varflagsexcl = d.getVar('BB_SIGNATURE_EXCLUDE_FLAGS')
 
     deps = {}
     values = {}
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/data_smart.py b/import-layers/yocto-poky/bitbake/lib/bb/data_smart.py
index 805a9a7..7dc1c68 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/data_smart.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/data_smart.py
@@ -108,7 +108,7 @@
                 varparse = self.d.expand_cache[key]
                 var = varparse.value
             else:
-                var = self.d.getVarFlag(key, "_content", True)
+                var = self.d.getVarFlag(key, "_content")
             self.references.add(key)
             if var is not None:
                 return var
@@ -116,13 +116,21 @@
                 return match.group()
 
     def python_sub(self, match):
-            code = match.group()[3:-1]
+            if isinstance(match, str):
+                code = match
+            else:
+                code = match.group()[3:-1]
+
+            if "_remote_data" in self.d:
+                connector = self.d["_remote_data"]
+                return connector.expandPythonRef(self.varname, code, self.d)
+
             codeobj = compile(code.strip(), self.varname or "<expansion>", "eval")
 
             parser = bb.codeparser.PythonParser(self.varname, logger)
             parser.parse_python(code)
             if self.varname:
-                vardeps = self.d.getVarFlag(self.varname, "vardeps", True)
+                vardeps = self.d.getVarFlag(self.varname, "vardeps")
                 if vardeps is None:
                     parser.log.flush()
             else:
@@ -146,7 +154,7 @@
         self['d'] = metadata
 
     def __missing__(self, key):
-        value = self.metadata.getVar(key, True)
+        value = self.metadata.getVar(key)
         if value is None or self.metadata.getVarFlag(key, 'func', False):
             raise KeyError(key)
         else:
@@ -222,6 +230,19 @@
         new.variables = self.variables.copy()
         return new
 
+    def __getstate__(self):
+        vardict = {}
+        for k, v in self.variables.iteritems():
+            vardict[k] = v
+        return {'dataroot': self.dataroot,
+                'variables': vardict}
+
+    def __setstate__(self, state):
+        self.dataroot = state['dataroot']
+        self.variables = COWDictBase.copy()
+        for k, v in state['variables'].items():
+            self.variables[k] = v
+
     def record(self, *kwonly, **loginfo):
         if not self.dataroot._tracking:
             return
@@ -247,10 +268,15 @@
         self.variables[var].append(loginfo.copy())
 
     def variable(self, var):
-        if var in self.variables:
-            return self.variables[var]
+        remote_connector = self.dataroot.getVar('_remote_data', False)
+        if remote_connector:
+            varhistory = remote_connector.getVarHistory(var)
         else:
-            return []
+            varhistory = []
+
+        if var in self.variables:
+            varhistory.extend(self.variables[var])
+        return varhistory
 
     def emit(self, var, oval, val, o, d):
         history = self.variable(var)
@@ -318,7 +344,7 @@
         the files in which they were added.
         """
         history = self.variable(var)
-        finalitems = (d.getVar(var, True) or '').split()
+        finalitems = (d.getVar(var) or '').split()
         filemap = {}
         isset = False
         for event in history:
@@ -426,11 +452,11 @@
             # Can end up here recursively so setup dummy values
             self.overrides = []
             self.overridesset = set()
-            self.overrides = (self.getVar("OVERRIDES", True) or "").split(":") or []
+            self.overrides = (self.getVar("OVERRIDES") or "").split(":") or []
             self.overridesset = set(self.overrides)
             self.inoverride = False
             self.expand_cache = {}
-            newoverrides = (self.getVar("OVERRIDES", True) or "").split(":") or []
+            newoverrides = (self.getVar("OVERRIDES") or "").split(":") or []
             if newoverrides == self.overrides:
                 break
             self.overrides = newoverrides
@@ -447,17 +473,22 @@
         dest = self.dict
         while dest:
             if var in dest:
-                return dest[var]
+                return dest[var], self.overridedata.get(var, None)
+
+            if "_remote_data" in dest:
+                connector = dest["_remote_data"]["_content"]
+                return connector.getVar(var)
 
             if "_data" not in dest:
                 break
             dest = dest["_data"]
+        return None, self.overridedata.get(var, None)
 
     def _makeShadowCopy(self, var):
         if var in self.dict:
             return
 
-        local_var = self._findVar(var)
+        local_var, _ = self._findVar(var)
 
         if local_var:
             self.dict[var] = copy.copy(local_var)
@@ -471,6 +502,12 @@
         if 'parsing' in loginfo:
             parsing=True
 
+        if '_remote_data' in self.dict:
+            connector = self.dict["_remote_data"]["_content"]
+            res = connector.setVar(var, value)
+            if not res:
+                return
+
         if 'op' not in loginfo:
             loginfo['op'] = "set"
         self.expand_cache = {}
@@ -509,6 +546,8 @@
                 del self.dict[var]["_append"]
             if "_prepend" in self.dict[var]:
                 del self.dict[var]["_prepend"]
+            if "_remove" in self.dict[var]:
+                del self.dict[var]["_remove"]
             if var in self.overridedata:
                 active = []
                 self.need_overrides()
@@ -541,7 +580,7 @@
             nextnew = set()
             self.overridevars.update(new)
             for i in new:
-                vardata = self.expandWithRefs(self.getVar(i, True), i)
+                vardata = self.expandWithRefs(self.getVar(i), i)
                 nextnew.update(vardata.references)
                 nextnew.update(vardata.contains.keys())
             new = nextnew
@@ -565,13 +604,19 @@
                 if len(shortvar) == 0:
                     override = None
 
-    def getVar(self, var, expand, noweakdefault=False, parsing=False):
+    def getVar(self, var, expand=True, noweakdefault=False, parsing=False):
         return self.getVarFlag(var, "_content", expand, noweakdefault, parsing)
 
     def renameVar(self, key, newkey, **loginfo):
         """
         Rename the variable key to newkey
         """
+        if '_remote_data' in self.dict:
+            connector = self.dict["_remote_data"]["_content"]
+            res = connector.renameVar(key, newkey)
+            if not res:
+                return
+
         val = self.getVar(key, 0, parsing=True)
         if val is not None:
             loginfo['variable'] = newkey
@@ -615,6 +660,12 @@
         self.setVar(var + "_prepend", value, ignore=True, parsing=True)
 
     def delVar(self, var, **loginfo):
+        if '_remote_data' in self.dict:
+            connector = self.dict["_remote_data"]["_content"]
+            res = connector.delVar(var)
+            if not res:
+                return
+
         loginfo['detail'] = ""
         loginfo['op'] = 'del'
         self.varhistory.record(**loginfo)
@@ -641,6 +692,12 @@
                          override = None
 
     def setVarFlag(self, var, flag, value, **loginfo):
+        if '_remote_data' in self.dict:
+            connector = self.dict["_remote_data"]["_content"]
+            res = connector.setVarFlag(var, flag, value)
+            if not res:
+                return
+
         self.expand_cache = {}
         if 'op' not in loginfo:
             loginfo['op'] = "set"
@@ -662,14 +719,14 @@
                 self.dict["__exportlist"]["_content"] = set()
             self.dict["__exportlist"]["_content"].add(var)
 
-    def getVarFlag(self, var, flag, expand, noweakdefault=False, parsing=False):
-        local_var = self._findVar(var)
+    def getVarFlag(self, var, flag, expand=True, noweakdefault=False, parsing=False):
+        local_var, overridedata = self._findVar(var)
         value = None
-        if flag == "_content" and var in self.overridedata and not parsing:
+        if flag == "_content" and overridedata is not None and not parsing:
             match = False
             active = {}
             self.need_overrides()
-            for (r, o) in self.overridedata[var]:
+            for (r, o) in overridedata:
                 # What about double overrides both with "_" in the name?
                 if o in self.overridesset:
                     active[o] = r
@@ -759,8 +816,14 @@
         return value
 
     def delVarFlag(self, var, flag, **loginfo):
+        if '_remote_data' in self.dict:
+            connector = self.dict["_remote_data"]["_content"]
+            res = connector.delVarFlag(var, flag)
+            if not res:
+                return
+
         self.expand_cache = {}
-        local_var = self._findVar(var)
+        local_var, _ = self._findVar(var)
         if not local_var:
             return
         if not var in self.dict:
@@ -803,7 +866,7 @@
             self.dict[var][i] = flags[i]
 
     def getVarFlags(self, var, expand = False, internalflags=False):
-        local_var = self._findVar(var)
+        local_var, _ = self._findVar(var)
         flags = {}
 
         if local_var:
@@ -845,7 +908,7 @@
         data = DataSmart()
         data.dict["_data"] = self.dict
         data.varhistory = self.varhistory.copy()
-        data.varhistory.datasmart = data
+        data.varhistory.dataroot = data
         data.inchistory = self.inchistory.copy()
 
         data._tracking = self._tracking
@@ -876,7 +939,7 @@
 
     def localkeys(self):
         for key in self.dict:
-            if key != '_data':
+            if key not in ['_data', '_remote_data']:
                 yield key
 
     def __iter__(self):
@@ -885,7 +948,7 @@
         def keylist(d):        
             klist = set()
             for key in d:
-                if key == "_data":
+                if key in ["_data", "_remote_data"]:
                     continue
                 if key in deleted:
                     continue
@@ -899,6 +962,13 @@
             if "_data" in d:
                 klist |= keylist(d["_data"])
 
+            if "_remote_data" in d:
+                connector = d["_remote_data"]["_content"]
+                for key in connector.getKeys():
+                    if key in deleted:
+                        continue
+                    klist.add(key)
+
             return klist
 
         self.need_overrides()
@@ -936,9 +1006,8 @@
         data = {}
         d = self.createCopy()
         bb.data.expandKeys(d)
-        bb.data.update_data(d)
 
-        config_whitelist = set((d.getVar("BB_HASHCONFIG_WHITELIST", True) or "").split())
+        config_whitelist = set((d.getVar("BB_HASHCONFIG_WHITELIST") or "").split())
         keys = set(key for key in iter(d) if not key.startswith("__"))
         for key in keys:
             if key in config_whitelist:
@@ -957,7 +1026,6 @@
 
         for key in ["__BBTASKS", "__BBANONFUNCS", "__BBHANDLERS"]:
             bb_list = d.getVar(key, False) or []
-            bb_list.sort()
             data.update({key:str(bb_list)})
 
             if key == "__BBANONFUNCS":
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/event.py b/import-layers/yocto-poky/bitbake/lib/bb/event.py
index 6f1cb10..6d8493b 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/event.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/event.py
@@ -48,6 +48,16 @@
     def __init__(self):
         self.pid = worker_pid
 
+
+class HeartbeatEvent(Event):
+    """Triggered at regular time intervals of 10 seconds. Other events can fire much more often
+       (runQueueTaskStarted when there are many short tasks) or not at all for long periods
+       of time (again runQueueTaskStarted, when there is just one long-running task), so this
+       event is more suitable for doing some task-independent work occassionally."""
+    def __init__(self, time):
+        Event.__init__(self)
+        self.time = time
+
 Registered        = 10
 AlreadyRegistered = 14
 
@@ -351,6 +361,17 @@
 class RecipePreFinalise(RecipeEvent):
     """ Recipe Parsing Complete but not yet finialised"""
 
+class RecipeTaskPreProcess(RecipeEvent):
+    """
+    Recipe Tasks about to be finalised
+    The list of tasks should be final at this point and handlers
+    are only able to change interdependencies
+    """
+    def __init__(self, fn, tasklist):
+        self.fn = fn
+        self.tasklist = tasklist
+        Event.__init__(self)
+
 class RecipeParsed(RecipeEvent):
     """ Recipe Parsing Complete """
 
@@ -372,7 +393,7 @@
     targets = property(getTargets)
 
 class BuildBase(Event):
-    """Base class for bbmake run events"""
+    """Base class for bitbake build events"""
 
     def __init__(self, n, p, failures = 0):
         self._name = n
@@ -417,13 +438,13 @@
         BuildBase.__init__(self, name, p)
 
 class BuildStarted(BuildBase, OperationStarted):
-    """bbmake build run started"""
+    """Event when builds start"""
     def __init__(self, n, p, failures = 0):
         OperationStarted.__init__(self, "Building Started")
         BuildBase.__init__(self, n, p, failures)
 
 class BuildCompleted(BuildBase, OperationCompleted):
-    """bbmake build run completed"""
+    """Event when builds have completed"""
     def __init__(self, total, n, p, failures=0, interrupted=0):
         if not failures:
             OperationCompleted.__init__(self, total, "Building Succeeded")
@@ -441,6 +462,23 @@
         self._free = freespace
         self._mountpoint = mountpoint
 
+class DiskUsageSample:
+    def __init__(self, available_bytes, free_bytes, total_bytes):
+        # Number of bytes available to non-root processes.
+        self.available_bytes = available_bytes
+        # Number of bytes available to root processes.
+        self.free_bytes = free_bytes
+        # Total capacity of the volume.
+        self.total_bytes = total_bytes
+
+class MonitorDiskEvent(Event):
+    """If BB_DISKMON_DIRS is set, then this event gets triggered each time disk space is checked.
+       Provides information about devices that are getting monitored."""
+    def __init__(self, disk_usage):
+        Event.__init__(self)
+        # hash of device root path -> DiskUsageSample
+        self.disk_usage = disk_usage
+
 class NoProvider(Event):
     """No Provider for an Event"""
 
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/__init__.py b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/__init__.py
index cd7362c..b853da3 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/__init__.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/__init__.py
@@ -35,9 +35,9 @@
 import collections
 import subprocess
 import pickle
+import errno
 import bb.persist_data, bb.utils
 import bb.checksum
-from bb import data
 import bb.process
 
 __version__ = "2"
@@ -355,7 +355,7 @@
     user, password, parameters).
     """
 
-    m = re.compile('(?P<type>[^:]*)://((?P<user>[^/]+)@)?(?P<location>[^;]+)(;(?P<parm>.*))?').match(url)
+    m = re.compile('(?P<type>[^:]*)://((?P<user>[^/;]+)@)?(?P<location>[^;]+)(;(?P<parm>.*))?').match(url)
     if not m:
         raise MalformedUrl(url)
 
@@ -491,7 +491,7 @@
     Calls before this must not hit the cache.
     """
     # When to drop SCM head revisions controlled by user policy
-    srcrev_policy = d.getVar('BB_SRCREV_POLICY', True) or "clear"
+    srcrev_policy = d.getVar('BB_SRCREV_POLICY') or "clear"
     if srcrev_policy == "cache":
         logger.debug(1, "Keeping SRCREV cache due to cache policy of: %s", srcrev_policy)
     elif srcrev_policy == "clear":
@@ -537,7 +537,11 @@
     return False
 
 def mirror_from_string(data):
-    return [ i.split() for i in (data or "").replace('\\n','\n').split('\n') if i ]
+    mirrors = (data or "").replace('\\n',' ').split()
+    # Split into pairs
+    if len(mirrors) % 2 != 0:
+        bb.warn('Invalid mirror data %s, should have paired members.' % data)
+    return list(zip(*[iter(mirrors)]*2))
 
 def verify_checksum(ud, d, precomputed={}):
     """
@@ -572,7 +576,7 @@
 
     if ud.method.recommends_checksum(ud) and not ud.md5_expected and not ud.sha256_expected:
         # If strict checking enabled and neither sum defined, raise error
-        strict = d.getVar("BB_STRICT_CHECKSUM", True) or "0"
+        strict = d.getVar("BB_STRICT_CHECKSUM") or "0"
         if strict == "1":
             logger.error('No checksum specified for %s, please add at least one to the recipe:\n'
                              'SRC_URI[%s] = "%s"\nSRC_URI[%s] = "%s"' %
@@ -621,7 +625,7 @@
     Returns True, if the donestamp exists and is valid, False otherwise. When
     returning False, any existing done stamps are removed.
     """
-    if not ud.needdonestamp:
+    if not ud.needdonestamp or (origud and not origud.needdonestamp):
         return True
 
     if not os.path.exists(ud.donestamp):
@@ -718,13 +722,13 @@
 
 def get_autorev(d):
     #  only not cache src rev in autorev case
-    if d.getVar('BB_SRCREV_POLICY', True) != "cache":
+    if d.getVar('BB_SRCREV_POLICY') != "cache":
         d.setVar('BB_DONT_CACHE', '1')
     return "AUTOINC"
 
 def get_srcrev(d, method_name='sortable_revision'):
     """
-    Return the revsion string, usually for use in the version string (PV) of the current package
+    Return the revision string, usually for use in the version string (PV) of the current package
     Most packages usually only have one SCM so we just pass on the call.
     In the multi SCM case, we build a value based on SRCREV_FORMAT which must
     have been set.
@@ -737,7 +741,7 @@
     """
 
     scms = []
-    fetcher = Fetch(d.getVar('SRC_URI', True).split(), d)
+    fetcher = Fetch(d.getVar('SRC_URI').split(), d)
     urldata = fetcher.ud
     for u in urldata:
         if urldata[u].method.supports_srcrev():
@@ -757,7 +761,7 @@
     #
     # Mutiple SCMs are in SRC_URI so we resort to SRCREV_FORMAT
     #
-    format = d.getVar('SRCREV_FORMAT', True)
+    format = d.getVar('SRCREV_FORMAT')
     if not format:
         raise FetchError("The SRCREV_FORMAT variable must be set when multiple SCMs are used.")
 
@@ -819,9 +823,18 @@
     if not cleanup:
         cleanup = []
 
+    # If PATH contains WORKDIR which contains PV which contains SRCPV we
+    # can end up in circular recursion here so give the option of breaking it
+    # in a data store copy.
+    try:
+        d.getVar("PV")
+    except bb.data_smart.ExpansionError:
+        d = bb.data.createCopy(d)
+        d.setVar("PV", "fetcheravoidrecurse")
+
     origenv = d.getVar("BB_ORIGENV", False)
     for var in exportvars:
-        val = d.getVar(var, True) or (origenv and origenv.getVar(var, True))
+        val = d.getVar(var) or (origenv and origenv.getVar(var))
         if val:
             cmd = 'export ' + var + '=\"%s\"; %s' % (val, cmd)
 
@@ -856,12 +869,15 @@
 
     return output
 
-def check_network_access(d, info = "", url = None):
+def check_network_access(d, info, url):
     """
-    log remote network access, and error if BB_NO_NETWORK is set
+    log remote network access, and error if BB_NO_NETWORK is set or the given
+    URI is untrusted
     """
-    if d.getVar("BB_NO_NETWORK", True) == "1":
+    if d.getVar("BB_NO_NETWORK") == "1":
         raise NetworkAccess(url, info)
+    elif not trusted_network(d, url):
+        raise UntrustedUrl(url, info)
     else:
         logger.debug(1, "Fetcher accessed the network with the command %s" % info)
 
@@ -958,7 +974,7 @@
 
         # We may be obtaining a mirror tarball which needs further processing by the real fetcher
         # If that tarball is a local file:// we need to provide a symlink to it
-        dldir = ld.getVar("DL_DIR", True)
+        dldir = ld.getVar("DL_DIR")
         if origud.mirrortarball and os.path.basename(ud.localpath) == os.path.basename(origud.mirrortarball) \
                 and os.path.basename(ud.localpath) != os.path.basename(origud.localpath):
             # Create donestamp in old format to avoid triggering a re-download
@@ -967,7 +983,14 @@
                 open(ud.donestamp, 'w').close()
             dest = os.path.join(dldir, os.path.basename(ud.localpath))
             if not os.path.exists(dest):
-                os.symlink(ud.localpath, dest)
+                # In case this is executing without any file locks held (as is
+                # the case for file:// URLs), two tasks may end up here at the
+                # same time, in which case we do not want the second task to
+                # fail when the link has already been created by the first task.
+                try:
+                    os.symlink(ud.localpath, dest)
+                except FileExistsError:
+                    pass
             if not verify_donestamp(origud, ld) or origud.method.need_update(origud, ld):
                 origud.method.download(origud, ld)
                 if hasattr(origud.method,"build_mirror_data"):
@@ -979,13 +1002,23 @@
                 # Broken symbolic link
                 os.unlink(origud.localpath)
 
-            os.symlink(ud.localpath, origud.localpath)
+            # As per above, in case two tasks end up here simultaneously.
+            try:
+                os.symlink(ud.localpath, origud.localpath)
+            except FileExistsError:
+                pass
         update_stamp(origud, ld)
         return ud.localpath
 
     except bb.fetch2.NetworkAccess:
         raise
 
+    except IOError as e:
+        if e.errno in [os.errno.ESTALE]:
+            logger.warn("Stale Error Observed %s." % ud.url)
+            return False
+        raise
+
     except bb.fetch2.BBFetchException as e:
         if isinstance(e, ChecksumError):
             logger.warning("Mirror checksum failure for url %s (original url: %s)\nCleaning and trying again." % (ud.url, origud.url))
@@ -1032,14 +1065,14 @@
     BB_ALLOWED_NETWORKS is set globally or for a specific recipe.
     Note: modifies SRC_URI & mirrors.
     """
-    if d.getVar('BB_NO_NETWORK', True) == "1":
+    if d.getVar('BB_NO_NETWORK') == "1":
         return True
 
     pkgname = d.expand(d.getVar('PN', False))
     trusted_hosts = d.getVarFlag('BB_ALLOWED_NETWORKS', pkgname, False)
 
     if not trusted_hosts:
-        trusted_hosts = d.getVar('BB_ALLOWED_NETWORKS', True)
+        trusted_hosts = d.getVar('BB_ALLOWED_NETWORKS')
 
     # Not enabled.
     if not trusted_hosts:
@@ -1071,7 +1104,7 @@
     """
 
     srcrev = None
-    pn = d.getVar("PN", True)
+    pn = d.getVar("PN")
     attempts = []
     if name != '' and pn:
         attempts.append("SRCREV_%s_pn-%s" % (name, pn))
@@ -1082,7 +1115,7 @@
     attempts.append("SRCREV")
 
     for a in attempts:
-        srcrev = d.getVar(a, True)              
+        srcrev = d.getVar(a)              
         if srcrev and srcrev != "INVALID":
             break
 
@@ -1115,7 +1148,7 @@
     """
     fetch = Fetch([], d, cache = False, localonly = True)
 
-    dl_dir = d.getVar('DL_DIR', True)
+    dl_dir = d.getVar('DL_DIR')
     filelist = []
     for u in fetch.urls:
         ud = fetch.ud[u]
@@ -1129,9 +1162,9 @@
                 if f.startswith(dl_dir):
                     # The local fetcher's behaviour is to return a path under DL_DIR if it couldn't find the file anywhere else
                     if os.path.exists(f):
-                        bb.warn("Getting checksum for %s SRC_URI entry %s: file not found except in DL_DIR" % (d.getVar('PN', True), os.path.basename(f)))
+                        bb.warn("Getting checksum for %s SRC_URI entry %s: file not found except in DL_DIR" % (d.getVar('PN'), os.path.basename(f)))
                     else:
-                        bb.warn("Unable to get checksum for %s SRC_URI entry %s: file could not be found" % (d.getVar('PN', True), os.path.basename(f)))
+                        bb.warn("Unable to get checksum for %s SRC_URI entry %s: file could not be found" % (d.getVar('PN'), os.path.basename(f)))
                 filelist.append(f + ":" + str(os.path.exists(f)))
 
     return " ".join(filelist)
@@ -1160,7 +1193,7 @@
         self.mirrortarball = None
         self.basename = None
         self.basepath = None
-        (self.type, self.host, self.path, self.user, self.pswd, self.parm) = decodeurl(data.expand(url, d))
+        (self.type, self.host, self.path, self.user, self.pswd, self.parm) = decodeurl(d.expand(url))
         self.date = self.getSRCDate(d)
         self.url = url
         if not self.user and "user" in self.parm:
@@ -1177,16 +1210,16 @@
             self.sha256_name = "sha256sum"
         if self.md5_name in self.parm:
             self.md5_expected = self.parm[self.md5_name]
-        elif self.type not in ["http", "https", "ftp", "ftps", "sftp"]:
+        elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3"]:
             self.md5_expected = None
         else:
-            self.md5_expected = d.getVarFlag("SRC_URI", self.md5_name, True)
+            self.md5_expected = d.getVarFlag("SRC_URI", self.md5_name)
         if self.sha256_name in self.parm:
             self.sha256_expected = self.parm[self.sha256_name]
-        elif self.type not in ["http", "https", "ftp", "ftps", "sftp"]:
+        elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3"]:
             self.sha256_expected = None
         else:
-            self.sha256_expected = d.getVarFlag("SRC_URI", self.sha256_name, True)
+            self.sha256_expected = d.getVarFlag("SRC_URI", self.sha256_name)
         self.ignore_checksums = False
 
         self.names = self.parm.get("name",'default').split(',')
@@ -1204,7 +1237,7 @@
             raise NonLocalMethod()
 
         if self.parm.get("proto", None) and "protocol" not in self.parm:
-            logger.warning('Consider updating %s recipe to use "protocol" not "proto" in SRC_URI.', d.getVar('PN', True))
+            logger.warning('Consider updating %s recipe to use "protocol" not "proto" in SRC_URI.', d.getVar('PN'))
             self.parm["protocol"] = self.parm.get("proto", None)
 
         if hasattr(self.method, "urldata_init"):
@@ -1217,7 +1250,7 @@
         elif self.localfile:
             self.localpath = self.method.localpath(self, d)
 
-        dldir = d.getVar("DL_DIR", True)
+        dldir = d.getVar("DL_DIR")
 
         if not self.needdonestamp:
             return
@@ -1235,7 +1268,7 @@
         self.donestamp = basepath + '.done'
         self.lockfile = basepath + '.lock'
 
-    def setup_revisons(self, d):
+    def setup_revisions(self, d):
         self.revisions = {}
         for name in self.names:
             self.revisions[name] = srcrev_internal_helper(self, d, name)
@@ -1257,12 +1290,12 @@
         if "srcdate" in self.parm:
             return self.parm['srcdate']
 
-        pn = d.getVar("PN", True)
+        pn = d.getVar("PN")
 
         if pn:
-            return d.getVar("SRCDATE_%s" % pn, True) or d.getVar("SRCDATE", True) or d.getVar("DATE", True)
+            return d.getVar("SRCDATE_%s" % pn) or d.getVar("SRCDATE") or d.getVar("DATE")
 
-        return d.getVar("SRCDATE", True) or d.getVar("DATE", True)
+        return d.getVar("SRCDATE") or d.getVar("DATE")
 
 class FetchMethod(object):
     """Base class for 'fetch'ing data"""
@@ -1282,7 +1315,7 @@
         Can also setup variables in urldata for use in go (saving code duplication
         and duplicate code execution)
         """
-        return os.path.join(data.getVar("DL_DIR", d, True), urldata.localfile)
+        return os.path.join(d.getVar("DL_DIR"), urldata.localfile)
 
     def supports_checksum(self, urldata):
         """
@@ -1382,6 +1415,10 @@
                 cmd = 'lzip -dc %s | tar x --no-same-owner -f -' % file
             elif file.endswith('.lz'):
                 cmd = 'lzip -dc %s > %s' % (file, efile)
+            elif file.endswith('.tar.7z'):
+                cmd = '7z x -so %s | tar x --no-same-owner -f -' % file
+            elif file.endswith('.7z'):
+                cmd = '7za x -y %s 1>/dev/null' % file
             elif file.endswith('.zip') or file.endswith('.jar'):
                 try:
                     dos = bb.utils.to_boolean(urldata.parm.get('dos'), False)
@@ -1413,10 +1450,6 @@
                 else:
                     raise UnpackError("Unable to unpack deb/ipk package - could not list contents", urldata.url)
                 cmd = 'ar x %s %s && tar --no-same-owner -xpf %s && rm %s' % (file, datafile, datafile, datafile)
-            elif file.endswith('.tar.7z'):
-                cmd = '7z x -so %s | tar xf - ' % file
-            elif file.endswith('.7z'):
-                cmd = '7za x -y %s 1>/dev/null' % file
 
         # If 'subdir' param exists, create a dir and use it as destination for unpack cmd
         if 'subdir' in urldata.parm:
@@ -1450,7 +1483,7 @@
         if not cmd:
             return
 
-        path = data.getVar('PATH', True)
+        path = data.getVar('PATH')
         if path:
             cmd = "PATH=\"%s\" %s" % (path, cmd)
         bb.note("Unpacking %s to %s/" % (file, unpackdir))
@@ -1507,7 +1540,7 @@
 
     def generate_revision_key(self, ud, d, name):
         key = self._revision_key(ud, d, name)
-        return "%s-%s" % (key, d.getVar("PN", True) or "")
+        return "%s-%s" % (key, d.getVar("PN") or "")
 
 class Fetch(object):
     def __init__(self, urls, d, cache = True, localonly = False, connection_cache = None):
@@ -1515,14 +1548,14 @@
             raise Exception("bb.fetch2.Fetch.__init__: cannot set cache and localonly at same time")
 
         if len(urls) == 0:
-            urls = d.getVar("SRC_URI", True).split()
+            urls = d.getVar("SRC_URI").split()
         self.urls = urls
         self.d = d
         self.ud = {}
         self.connection_cache = connection_cache
 
-        fn = d.getVar('FILE', True)
-        mc = d.getVar('__BBMULTICONFIG', True) or ""
+        fn = d.getVar('FILE')
+        mc = d.getVar('__BBMULTICONFIG') or ""
         if cache and fn and mc + fn in urldata_cache:
             self.ud = urldata_cache[mc + fn]
 
@@ -1565,8 +1598,8 @@
         if not urls:
             urls = self.urls
 
-        network = self.d.getVar("BB_NO_NETWORK", True)
-        premirroronly = (self.d.getVar("BB_FETCH_PREMIRRORONLY", True) == "1")
+        network = self.d.getVar("BB_NO_NETWORK")
+        premirroronly = (self.d.getVar("BB_FETCH_PREMIRRORONLY") == "1")
 
         for u in urls:
             ud = self.ud[u]
@@ -1584,8 +1617,17 @@
                     localpath = ud.localpath
                 elif m.try_premirror(ud, self.d):
                     logger.debug(1, "Trying PREMIRRORS")
-                    mirrors = mirror_from_string(self.d.getVar('PREMIRRORS', True))
+                    mirrors = mirror_from_string(self.d.getVar('PREMIRRORS'))
                     localpath = try_mirrors(self, self.d, ud, mirrors, False)
+                    if localpath:
+                        try:
+                            # early checksum verification so that if the checksum of the premirror
+                            # contents mismatch the fetcher can still try upstream and mirrors
+                            update_stamp(ud, self.d)
+                        except ChecksumError as e:
+                            logger.warning("Checksum failure encountered with premirror download of %s - will attempt other sources." % u)
+                            logger.debug(1, str(e))
+                            localpath = ""
 
                 if premirroronly:
                     self.d.setVar("BB_NO_NETWORK", "1")
@@ -1624,7 +1666,7 @@
                         if not verified_stamp:
                             m.clean(ud, self.d)
                         logger.debug(1, "Trying MIRRORS")
-                        mirrors = mirror_from_string(self.d.getVar('MIRRORS', True))
+                        mirrors = mirror_from_string(self.d.getVar('MIRRORS'))
                         localpath = try_mirrors(self, self.d, ud, mirrors)
 
                 if not localpath or ((not os.path.exists(localpath)) and localpath.find("*") == -1):
@@ -1634,6 +1676,11 @@
 
                 update_stamp(ud, self.d)
 
+            except IOError as e:
+                if e.errno in [os.errno.ESTALE]:
+                    logger.error("Stale Error Observed %s." % u)
+                    raise ChecksumError("Stale Error Detected")
+
             except BBFetchException as e:
                 if isinstance(e, ChecksumError):
                     logger.error("Checksum failure fetching %s" % u)
@@ -1657,7 +1704,7 @@
             m = ud.method
             logger.debug(1, "Testing URL %s", u)
             # First try checking uri, u, from PREMIRRORS
-            mirrors = mirror_from_string(self.d.getVar('PREMIRRORS', True))
+            mirrors = mirror_from_string(self.d.getVar('PREMIRRORS'))
             ret = try_mirrors(self, self.d, ud, mirrors, True)
             if not ret:
                 # Next try checking from the original uri, u
@@ -1665,7 +1712,7 @@
                     ret = m.checkstatus(self, ud, self.d)
                 except:
                     # Finally, try checking uri, u, from MIRRORS
-                    mirrors = mirror_from_string(self.d.getVar('MIRRORS', True))
+                    mirrors = mirror_from_string(self.d.getVar('MIRRORS'))
                     ret = try_mirrors(self, self.d, ud, mirrors, True)
 
             if not ret:
@@ -1763,6 +1810,7 @@
 from . import wget
 from . import ssh
 from . import sftp
+from . import s3
 from . import perforce
 from . import bzr
 from . import hg
@@ -1780,6 +1828,7 @@
 methods.append(cvs.Cvs())
 methods.append(ssh.SSH())
 methods.append(sftp.SFTP())
+methods.append(s3.S3())
 methods.append(perforce.Perforce())
 methods.append(bzr.Bzr())
 methods.append(hg.Hg())
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/bzr.py b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/bzr.py
index 72264af..16123f8 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/bzr.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/bzr.py
@@ -27,7 +27,6 @@
 import sys
 import logging
 import bb
-from bb import data
 from bb.fetch2 import FetchMethod
 from bb.fetch2 import FetchError
 from bb.fetch2 import runfetchcmd
@@ -43,14 +42,14 @@
         """
         # Create paths to bzr checkouts
         relpath = self._strip_leading_slashes(ud.path)
-        ud.pkgdir = os.path.join(data.expand('${BZRDIR}', d), ud.host, relpath)
+        ud.pkgdir = os.path.join(d.expand('${BZRDIR}'), ud.host, relpath)
 
-        ud.setup_revisons(d)
+        ud.setup_revisions(d)
 
         if not ud.revision:
             ud.revision = self.latest_revision(ud, d)
 
-        ud.localfile = data.expand('bzr_%s_%s_%s.tar.gz' % (ud.host, ud.path.replace('/', '.'), ud.revision), d)
+        ud.localfile = d.expand('bzr_%s_%s_%s.tar.gz' % (ud.host, ud.path.replace('/', '.'), ud.revision))
 
     def _buildbzrcommand(self, ud, d, command):
         """
@@ -58,7 +57,7 @@
         command is "fetch", "update", "revno"
         """
 
-        basecmd = data.expand('${FETCHCMD_bzr}', d)
+        basecmd = d.expand('${FETCHCMD_bzr}')
 
         proto =  ud.parm.get('protocol', 'http')
 
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/clearcase.py b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/clearcase.py
index 70e280a..36beab6 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/clearcase.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/clearcase.py
@@ -65,7 +65,6 @@
 import sys
 import shutil
 import bb
-from   bb import data
 from   bb.fetch2 import FetchMethod
 from   bb.fetch2 import FetchError
 from   bb.fetch2 import runfetchcmd
@@ -108,13 +107,13 @@
         else:
             ud.module = ""
 
-        ud.basecmd = d.getVar("FETCHCMD_ccrc", True) or spawn.find_executable("cleartool") or spawn.find_executable("rcleartool")
+        ud.basecmd = d.getVar("FETCHCMD_ccrc") or spawn.find_executable("cleartool") or spawn.find_executable("rcleartool")
 
-        if data.getVar("SRCREV", d, True) == "INVALID":
+        if d.getVar("SRCREV") == "INVALID":
           raise FetchError("Set a valid SRCREV for the clearcase fetcher in your recipe, e.g. SRCREV = \"/main/LATEST\" or any other label of your choice.")
 
         ud.label = d.getVar("SRCREV", False)
-        ud.customspec = d.getVar("CCASE_CUSTOM_CONFIG_SPEC", True)
+        ud.customspec = d.getVar("CCASE_CUSTOM_CONFIG_SPEC")
 
         ud.server     = "%s://%s%s" % (ud.proto, ud.host, ud.path)
 
@@ -124,7 +123,7 @@
 
         ud.viewname         = "%s-view%s" % (ud.identifier, d.getVar("DATETIME", d, True))
         ud.csname           = "%s-config-spec" % (ud.identifier)
-        ud.ccasedir         = os.path.join(data.getVar("DL_DIR", d, True), ud.type)
+        ud.ccasedir         = os.path.join(d.getVar("DL_DIR"), ud.type)
         ud.viewdir          = os.path.join(ud.ccasedir, ud.viewname)
         ud.configspecfile   = os.path.join(ud.ccasedir, ud.csname)
         ud.localfile        = "%s.tar.gz" % (ud.identifier)
@@ -144,7 +143,7 @@
         self.debug("configspecfile  = %s" % ud.configspecfile)
         self.debug("localfile       = %s" % ud.localfile)
 
-        ud.localfile = os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
+        ud.localfile = os.path.join(d.getVar("DL_DIR"), ud.localfile)
 
     def _build_ccase_command(self, ud, command):
         """
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/cvs.py b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/cvs.py
index 5ff70ba..490c954 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/cvs.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/cvs.py
@@ -63,7 +63,7 @@
         if 'fullpath' in ud.parm:
             fullpath = '_fullpath'
 
-        ud.localfile = bb.data.expand('%s_%s_%s_%s%s%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.tag, ud.date, norecurse, fullpath), d)
+        ud.localfile = d.expand('%s_%s_%s_%s%s%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.tag, ud.date, norecurse, fullpath))
 
     def need_update(self, ud, d):
         if (ud.date == "now"):
@@ -87,10 +87,10 @@
             cvsroot = ud.path
         else:
             cvsroot = ":" + method
-            cvsproxyhost = d.getVar('CVS_PROXY_HOST', True)
+            cvsproxyhost = d.getVar('CVS_PROXY_HOST')
             if cvsproxyhost:
                 cvsroot += ";proxy=" + cvsproxyhost
-            cvsproxyport = d.getVar('CVS_PROXY_PORT', True)
+            cvsproxyport = d.getVar('CVS_PROXY_PORT')
             if cvsproxyport:
                 cvsroot += ";proxyport=" + cvsproxyport
             cvsroot += ":" + ud.user
@@ -110,7 +110,7 @@
         if ud.tag:
             options.append("-r %s" % ud.tag)
 
-        cvsbasecmd = d.getVar("FETCHCMD_cvs", True)
+        cvsbasecmd = d.getVar("FETCHCMD_cvs")
         cvscmd = cvsbasecmd + " '-d" + cvsroot + "' co " + " ".join(options) + " " + ud.module
         cvsupdatecmd = cvsbasecmd + " '-d" + cvsroot + "' update -d -P " + " ".join(options)
 
@@ -120,8 +120,8 @@
 
         # create module directory
         logger.debug(2, "Fetch: checking for module directory")
-        pkg = d.getVar('PN', True)
-        pkgdir = os.path.join(d.getVar('CVSDIR', True), pkg)
+        pkg = d.getVar('PN')
+        pkgdir = os.path.join(d.getVar('CVSDIR'), pkg)
         moddir = os.path.join(pkgdir, localdir)
         workdir = None
         if os.access(os.path.join(moddir, 'CVS'), os.R_OK):
@@ -164,8 +164,8 @@
     def clean(self, ud, d):
         """ Clean CVS Files and tarballs """
 
-        pkg = d.getVar('PN', True)
-        pkgdir = os.path.join(d.getVar("CVSDIR", True), pkg)
+        pkg = d.getVar('PN')
+        pkgdir = os.path.join(d.getVar("CVSDIR"), pkg)
 
         bb.utils.remove(pkgdir, True)
         bb.utils.remove(ud.localpath)
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/git.py b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/git.py
index 792c183..7442f84 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/git.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/git.py
@@ -50,7 +50,7 @@
    The default is "0", set nobranch=1 if needed.
 
 - usehead
-   For local git:// urls to use the current branch HEAD as the revsion for use with
+   For local git:// urls to use the current branch HEAD as the revision for use with
    AUTOREV. Implies nobranch.
 
 """
@@ -76,7 +76,6 @@
 import bb
 import errno
 import bb.progress
-from   bb    import data
 from   bb.fetch2 import FetchMethod
 from   bb.fetch2 import runfetchcmd
 from   bb.fetch2 import logger
@@ -174,19 +173,19 @@
         if len(branches) != len(ud.names):
             raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url)
         ud.branches = {}
-        for name in ud.names:
-            branch = branches[ud.names.index(name)]
+        for pos, name in enumerate(ud.names):
+            branch = branches[pos]
             ud.branches[name] = branch
             ud.unresolvedrev[name] = branch
 
         if ud.usehead:
             ud.unresolvedrev['default'] = 'HEAD'
 
-        ud.basecmd = data.getVar("FETCHCMD_git", d, True) or "git -c core.fsyncobjectfiles=0"
+        ud.basecmd = d.getVar("FETCHCMD_git") or "git -c core.fsyncobjectfiles=0"
 
-        ud.write_tarballs = ((data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True) or "0") != "0") or ud.rebaseable
+        ud.write_tarballs = ((d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0") != "0") or ud.rebaseable
 
-        ud.setup_revisons(d)
+        ud.setup_revisions(d)
 
         for name in ud.names:
             # Ensure anything that doesn't look like a sha256 checksum/revision is translated into one
@@ -206,9 +205,9 @@
         if ud.rebaseable:
             for name in ud.names:
                 gitsrcname = gitsrcname + '_' + ud.revisions[name]
-        ud.mirrortarball = 'git2_%s.tar.gz' % (gitsrcname)
-        ud.fullmirror = os.path.join(d.getVar("DL_DIR", True), ud.mirrortarball)
-        gitdir = d.getVar("GITDIR", True) or (d.getVar("DL_DIR", True) + "/git2/")
+        ud.mirrortarball = 'git2_%s.tar.gz' % gitsrcname
+        ud.fullmirror = os.path.join(d.getVar("DL_DIR"), ud.mirrortarball)
+        gitdir = d.getVar("GITDIR") or (d.getVar("DL_DIR") + "/git2/")
         ud.clonedir = os.path.join(gitdir, gitsrcname)
 
         ud.localfile = ud.clonedir
@@ -229,7 +228,7 @@
     def try_premirror(self, ud, d):
         # If we don't do this, updating an existing checkout with only premirrors
         # is not possible
-        if d.getVar("BB_FETCH_PREMIRRORONLY", True) is not None:
+        if d.getVar("BB_FETCH_PREMIRRORONLY") is not None:
             return True
         if os.path.exists(ud.clonedir):
             return False
@@ -241,7 +240,7 @@
         # If the checkout doesn't exist and the mirror tarball does, extract it
         if not os.path.exists(ud.clonedir) and os.path.exists(ud.fullmirror):
             bb.utils.mkdirhier(ud.clonedir)
-            runfetchcmd("tar -xzf %s" % (ud.fullmirror), d, workdir=ud.clonedir)
+            runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=ud.clonedir)
 
         repourl = self._get_repo_url(ud)
 
@@ -252,7 +251,7 @@
                 repourl = repourl[7:]
             clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, repourl, ud.clonedir)
             if ud.proto.lower() != 'file':
-                bb.fetch2.check_network_access(d, clone_cmd)
+                bb.fetch2.check_network_access(d, clone_cmd, ud.url)
             progresshandler = GitProgressHandler(d)
             runfetchcmd(clone_cmd, d, log=progresshandler)
 
@@ -292,15 +291,15 @@
                 os.unlink(ud.fullmirror)
 
             logger.info("Creating tarball of git repository")
-            runfetchcmd("tar -czf %s %s" % (ud.fullmirror, os.path.join(".") ), d, workdir=ud.clonedir)
-            runfetchcmd("touch %s.done" % (ud.fullmirror), d, workdir=ud.clonedir)
+            runfetchcmd("tar -czf %s ." % ud.fullmirror, d, workdir=ud.clonedir)
+            runfetchcmd("touch %s.done" % ud.fullmirror, d)
 
     def unpack(self, ud, destdir, d):
         """ unpack the downloaded src to destdir"""
 
         subdir = ud.parm.get("subpath", "")
         if subdir != "":
-            readpathspec = ":%s" % (subdir)
+            readpathspec = ":%s" % subdir
             def_destsuffix = "%s/" % os.path.basename(subdir.rstrip('/'))
         else:
             readpathspec = ""
@@ -380,14 +379,26 @@
         """
         Run git ls-remote with the specified search string
         """
-        repourl = self._get_repo_url(ud)
-        cmd = "%s ls-remote %s %s" % \
-              (ud.basecmd, repourl, search)
-        if ud.proto.lower() != 'file':
-            bb.fetch2.check_network_access(d, cmd)
-        output = runfetchcmd(cmd, d, True)
-        if not output:
-            raise bb.fetch2.FetchError("The command %s gave empty output unexpectedly" % cmd, ud.url)
+        # Prevent recursion e.g. in OE if SRCPV is in PV, PV is in WORKDIR,
+        # and WORKDIR is in PATH (as a result of RSS), our call to
+        # runfetchcmd() exports PATH so this function will get called again (!)
+        # In this scenario the return call of the function isn't actually
+        # important - WORKDIR isn't needed in PATH to call git ls-remote
+        # anyway.
+        if d.getVar('_BB_GIT_IN_LSREMOTE', False):
+            return ''
+        d.setVar('_BB_GIT_IN_LSREMOTE', '1')
+        try:
+            repourl = self._get_repo_url(ud)
+            cmd = "%s ls-remote %s %s" % \
+                (ud.basecmd, repourl, search)
+            if ud.proto.lower() != 'file':
+                bb.fetch2.check_network_access(d, cmd, repourl)
+            output = runfetchcmd(cmd, d, True)
+            if not output:
+                raise bb.fetch2.FetchError("The command %s gave empty output unexpectedly" % cmd, ud.url)
+        finally:
+            d.delVar('_BB_GIT_IN_LSREMOTE')
         return output
 
     def _latest_revision(self, ud, d, name):
@@ -418,7 +429,7 @@
         """
         pupver = ('', '')
 
-        tagregex = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX', True) or "(?P<pver>([0-9][\.|_]?)+)")
+        tagregex = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX') or "(?P<pver>([0-9][\.|_]?)+)")
         try:
             output = self._lsremote(ud, d, "refs/tags/*")
         except bb.fetch2.FetchError or bb.fetch2.NetworkAccess:
@@ -470,7 +481,7 @@
             if not os.path.exists(rev_file) or not os.path.getsize(rev_file):
                 from pipes import quote
                 commits = bb.fetch2.runfetchcmd(
-                        "git rev-list %s -- | wc -l" % (quote(rev)),
+                        "git rev-list %s -- | wc -l" % quote(rev),
                         d, quiet=True).strip().lstrip('0')
                 if commits:
                     open(rev_file, "w").write("%d\n" % int(commits))
@@ -485,5 +496,5 @@
         try:
             self._lsremote(ud, d, "")
             return True
-        except FetchError:
+        except bb.fetch2.FetchError:
             return False
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/gitannex.py b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/gitannex.py
index 4937a10..c66c211 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/gitannex.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/gitannex.py
@@ -22,7 +22,6 @@
 
 import os
 import bb
-from   bb import data
 from   bb.fetch2.git import Git
 from   bb.fetch2 import runfetchcmd
 from   bb.fetch2 import logger
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/gitsm.py b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/gitsm.py
index 6613762..a95584c 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/gitsm.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/gitsm.py
@@ -31,7 +31,6 @@
 
 import os
 import bb
-from   bb    import data
 from   bb.fetch2.git import Git
 from   bb.fetch2 import runfetchcmd
 from   bb.fetch2 import logger
@@ -108,7 +107,7 @@
         os.rename(ud.clonedir, gitdir)
         runfetchcmd("sed " + gitdir + "/config -i -e 's/bare.*=.*true/bare = false/'", d)
         runfetchcmd(ud.basecmd + " reset --hard", d, workdir=tmpclonedir)
-        runfetchcmd(ud.basecmd + " checkout " + ud.revisions[ud.names[0]], d, workdir=tmpclonedir)
+        runfetchcmd(ud.basecmd + " checkout -f " + ud.revisions[ud.names[0]], d, workdir=tmpclonedir)
         runfetchcmd(ud.basecmd + " submodule update --init --recursive", d, workdir=tmpclonedir)
         self._set_relative_paths(tmpclonedir)
         runfetchcmd("sed " + gitdir + "/config -i -e 's/bare.*=.*false/bare = true/'", d, workdir=tmpclonedir)
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/hg.py b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/hg.py
index 20df801..b5f2686 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/hg.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/hg.py
@@ -29,7 +29,6 @@
 import logging
 import bb
 import errno
-from bb import data
 from bb.fetch2 import FetchMethod
 from bb.fetch2 import FetchError
 from bb.fetch2 import MissingParameterError
@@ -67,7 +66,7 @@
         else:
             ud.proto = "hg"
 
-        ud.setup_revisons(d)
+        ud.setup_revisions(d)
 
         if 'rev' in ud.parm:
             ud.revision = ud.parm['rev']
@@ -78,15 +77,15 @@
         hgsrcname = '%s_%s_%s' % (ud.module.replace('/', '.'), \
                             ud.host, ud.path.replace('/', '.'))
         ud.mirrortarball = 'hg_%s.tar.gz' % hgsrcname
-        ud.fullmirror = os.path.join(d.getVar("DL_DIR", True), ud.mirrortarball)
+        ud.fullmirror = os.path.join(d.getVar("DL_DIR"), ud.mirrortarball)
 
-        hgdir = d.getVar("HGDIR", True) or (d.getVar("DL_DIR", True) + "/hg/")
+        hgdir = d.getVar("HGDIR") or (d.getVar("DL_DIR") + "/hg/")
         ud.pkgdir = os.path.join(hgdir, hgsrcname)
         ud.moddir = os.path.join(ud.pkgdir, ud.module)
         ud.localfile = ud.moddir
-        ud.basecmd = data.getVar("FETCHCMD_hg", d, True) or "/usr/bin/env hg"
+        ud.basecmd = d.getVar("FETCHCMD_hg") or "/usr/bin/env hg"
 
-        ud.write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS", True)
+        ud.write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS")
 
     def need_update(self, ud, d):
         revTag = ud.parm.get('rev', 'tip')
@@ -99,7 +98,7 @@
     def try_premirror(self, ud, d):
         # If we don't do this, updating an existing checkout with only premirrors
         # is not possible
-        if d.getVar("BB_FETCH_PREMIRRORONLY", True) is not None:
+        if d.getVar("BB_FETCH_PREMIRRORONLY") is not None:
             return True
         if os.path.exists(ud.moddir):
             return False
@@ -221,7 +220,7 @@
         """
         Compute tip revision for the url
         """
-        bb.fetch2.check_network_access(d, self._buildhgcommand(ud, d, "info"))
+        bb.fetch2.check_network_access(d, self._buildhgcommand(ud, d, "info"), ud.url)
         output = runfetchcmd(self._buildhgcommand(ud, d, "info"), d)
         return output.strip()
 
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/local.py b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/local.py
index 51ca78d..a114ac1 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/local.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/local.py
@@ -29,7 +29,6 @@
 import urllib.request, urllib.parse, urllib.error
 import bb
 import bb.utils
-from   bb import data
 from   bb.fetch2 import FetchMethod, FetchError
 from   bb.fetch2 import logger
 
@@ -63,17 +62,11 @@
         newpath = path
         if path[0] == "/":
             return [path]
-        filespath = data.getVar('FILESPATH', d, True)
+        filespath = d.getVar('FILESPATH')
         if filespath:
             logger.debug(2, "Searching for %s in paths:\n    %s" % (path, "\n    ".join(filespath.split(":"))))
             newpath, hist = bb.utils.which(filespath, path, history=True)
             searched.extend(hist)
-        if not newpath:
-            filesdir = data.getVar('FILESDIR', d, True)
-            if filesdir:
-                logger.debug(2, "Searching for %s in path: %s" % (path, filesdir))
-                newpath = os.path.join(filesdir, path)
-                searched.append(newpath)
         if (not newpath or not os.path.exists(newpath)) and path.find("*") != -1:
             # For expressions using '*', best we can do is take the first directory in FILESPATH that exists
             newpath, hist = bb.utils.which(filespath, ".", history=True)
@@ -81,7 +74,7 @@
             logger.debug(2, "Searching for %s in path: %s" % (path, newpath))
             return searched
         if not os.path.exists(newpath):
-            dldirfile = os.path.join(d.getVar("DL_DIR", True), path)
+            dldirfile = os.path.join(d.getVar("DL_DIR"), path)
             logger.debug(2, "Defaulting to %s for %s" % (dldirfile, path))
             bb.utils.mkdirhier(os.path.dirname(dldirfile))
             searched.append(dldirfile)
@@ -100,13 +93,10 @@
         # no need to fetch local files, we'll deal with them in place.
         if self.supports_checksum(urldata) and not os.path.exists(urldata.localpath):
             locations = []
-            filespath = data.getVar('FILESPATH', d, True)
+            filespath = d.getVar('FILESPATH')
             if filespath:
                 locations = filespath.split(":")
-            filesdir = data.getVar('FILESDIR', d, True)
-            if filesdir:
-                locations.append(filesdir)
-            locations.append(d.getVar("DL_DIR", True))
+            locations.append(d.getVar("DL_DIR"))
 
             msg = "Unable to find file " + urldata.url + " anywhere. The paths that were searched were:\n    " + "\n    ".join(locations)
             raise FetchError(msg)
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/npm.py b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/npm.py
index 699ae72..73a75fe 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/npm.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/npm.py
@@ -25,7 +25,6 @@
 import subprocess
 import signal
 import bb
-from   bb import data
 from   bb.fetch2 import FetchMethod
 from   bb.fetch2 import FetchError
 from   bb.fetch2 import ChecksumError
@@ -80,6 +79,7 @@
         if not ud.version:
             raise ParameterError("NPM fetcher requires a version parameter", ud.url)
         ud.bbnpmmanifest = "%s-%s.deps.json" % (ud.pkgname, ud.version)
+        ud.bbnpmmanifest = ud.bbnpmmanifest.replace('/', '-')
         ud.registry = "http://%s" % (ud.url.replace('npm://', '', 1).split(';'))[0]
         prefixdir = "npm/%s" % ud.pkgname
         ud.pkgdatadir = d.expand("${DL_DIR}/%s" % prefixdir)
@@ -87,12 +87,13 @@
             bb.utils.mkdirhier(ud.pkgdatadir)
         ud.localpath = d.expand("${DL_DIR}/npm/%s" % ud.bbnpmmanifest)
 
-        self.basecmd = d.getVar("FETCHCMD_wget", True) or "/usr/bin/env wget -O -t 2 -T 30 -nv --passive-ftp --no-check-certificate "
+        self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -O -t 2 -T 30 -nv --passive-ftp --no-check-certificate "
         ud.prefixdir = prefixdir
 
-        ud.write_tarballs = ((data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True) or "0") != "0")
+        ud.write_tarballs = ((d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0") != "0")
         ud.mirrortarball = 'npm_%s-%s.tar.xz' % (ud.pkgname, ud.version)
-        ud.fullmirror = os.path.join(d.getVar("DL_DIR", True), ud.mirrortarball)
+        ud.mirrortarball = ud.mirrortarball.replace('/', '-')
+        ud.fullmirror = os.path.join(d.getVar("DL_DIR"), ud.mirrortarball)
 
     def need_update(self, ud, d):
         if os.path.exists(ud.localpath):
@@ -101,8 +102,8 @@
 
     def _runwget(self, ud, d, command, quiet):
         logger.debug(2, "Fetching %s using command '%s'" % (ud.url, command))
-        bb.fetch2.check_network_access(d, command)
-        dldir = d.getVar("DL_DIR", True)
+        bb.fetch2.check_network_access(d, command, ud.url)
+        dldir = d.getVar("DL_DIR")
         runfetchcmd(command, d, quiet, workdir=dldir)
 
     def _unpackdep(self, ud, pkg, data, destdir, dldir, d):
@@ -116,7 +117,7 @@
         # Change to subdir before executing command
         if not os.path.exists(destdir):
             os.makedirs(destdir)
-        path = d.getVar('PATH', True)
+        path = d.getVar('PATH')
         if path:
             cmd = "PATH=\"%s\" %s" % (path, cmd)
         bb.note("Unpacking %s to %s/" % (file, destdir))
@@ -132,9 +133,8 @@
 
 
     def unpack(self, ud, destdir, d):
-        dldir = d.getVar("DL_DIR", True)
-        depdumpfile = "%s-%s.deps.json" % (ud.pkgname, ud.version)
-        with open("%s/npm/%s" % (dldir, depdumpfile)) as datafile:
+        dldir = d.getVar("DL_DIR")
+        with open("%s/npm/%s" % (dldir, ud.bbnpmmanifest)) as datafile:
             workobj = json.load(datafile)
         dldir = "%s/%s" % (os.path.dirname(ud.localpath), ud.pkgname)
 
@@ -182,7 +182,12 @@
             if pkg_os:
                 if not isinstance(pkg_os, list):
                     pkg_os = [pkg_os]
-                if 'linux' not in pkg_os or '!linux' in pkg_os:
+                blacklist = False
+                for item in pkg_os:
+                    if item.startswith('!'):
+                        blacklist = True
+                        break
+                if (not blacklist and 'linux' not in pkg_os) or '!linux' in pkg_os:
                     logger.debug(2, "Skipping %s since it's incompatible with Linux" % pkg)
                     return
         #logger.debug(2, "Output URL is %s - %s - %s" % (ud.basepath, ud.basename, ud.localfile))
@@ -195,6 +200,7 @@
 
         dependencies = pdata.get('dependencies', {})
         optionalDependencies = pdata.get('optionalDependencies', {})
+        dependencies.update(optionalDependencies)
         depsfound = {}
         optdepsfound = {}
         data[pkg]['deps'] = {}
@@ -251,24 +257,30 @@
         lockdown = {}
 
         if not os.listdir(ud.pkgdatadir) and os.path.exists(ud.fullmirror):
-            dest = d.getVar("DL_DIR", True)
+            dest = d.getVar("DL_DIR")
             bb.utils.mkdirhier(dest)
             runfetchcmd("tar -xJf %s" % (ud.fullmirror), d, workdir=dest)
             return
 
-        shwrf = d.getVar('NPM_SHRINKWRAP', True)
+        shwrf = d.getVar('NPM_SHRINKWRAP')
         logger.debug(2, "NPM shrinkwrap file is %s" % shwrf)
-        try:
-            with open(shwrf) as datafile:
-                shrinkobj = json.load(datafile)
-        except:
+        if shwrf:
+            try:
+                with open(shwrf) as datafile:
+                    shrinkobj = json.load(datafile)
+            except Exception as e:
+                raise FetchError('Error loading NPM_SHRINKWRAP file "%s" for %s: %s' % (shwrf, ud.pkgname, str(e)))
+        elif not ud.ignore_checksums:
             logger.warning('Missing shrinkwrap file in NPM_SHRINKWRAP for %s, this will lead to unreliable builds!' % ud.pkgname)
-        lckdf = d.getVar('NPM_LOCKDOWN', True)
+        lckdf = d.getVar('NPM_LOCKDOWN')
         logger.debug(2, "NPM lockdown file is %s" % lckdf)
-        try:
-            with open(lckdf) as datafile:
-                lockdown = json.load(datafile)
-        except:
+        if lckdf:
+            try:
+                with open(lckdf) as datafile:
+                    lockdown = json.load(datafile)
+            except Exception as e:
+                raise FetchError('Error loading NPM_LOCKDOWN file "%s" for %s: %s' % (lckdf, ud.pkgname, str(e)))
+        elif not ud.ignore_checksums:
             logger.warning('Missing lockdown file in NPM_LOCKDOWN for %s, this will lead to unreproducible builds!' % ud.pkgname)
 
         if ('name' not in shrinkobj):
@@ -286,7 +298,7 @@
             if os.path.islink(ud.fullmirror):
                 os.unlink(ud.fullmirror)
 
-            dldir = d.getVar("DL_DIR", True)
+            dldir = d.getVar("DL_DIR")
             logger.info("Creating tarball of npm data")
             runfetchcmd("tar -cJf %s npm/%s npm/%s" % (ud.fullmirror, ud.bbnpmmanifest, ud.pkgname), d,
                         workdir=dldir)
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/osc.py b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/osc.py
index 295abf9..2b4f7d9 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/osc.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/osc.py
@@ -10,7 +10,6 @@
 import  sys
 import logging
 import  bb
-from    bb       import data
 from    bb.fetch2 import FetchMethod
 from    bb.fetch2 import FetchError
 from    bb.fetch2 import MissingParameterError
@@ -34,7 +33,7 @@
 
         # Create paths to osc checkouts
         relpath = self._strip_leading_slashes(ud.path)
-        ud.pkgdir = os.path.join(d.getVar('OSCDIR', True), ud.host)
+        ud.pkgdir = os.path.join(d.getVar('OSCDIR'), ud.host)
         ud.moddir = os.path.join(ud.pkgdir, relpath, ud.module)
 
         if 'rev' in ud.parm:
@@ -47,7 +46,7 @@
             else:
                 ud.revision = ""
 
-        ud.localfile = data.expand('%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.path.replace('/', '.'), ud.revision), d)
+        ud.localfile = d.expand('%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.path.replace('/', '.'), ud.revision))
 
     def _buildosccommand(self, ud, d, command):
         """
@@ -55,7 +54,7 @@
         command is "fetch", "update", "info"
         """
 
-        basecmd = data.expand('${FETCHCMD_osc}', d)
+        basecmd = d.expand('${FETCHCMD_osc}')
 
         proto = ud.parm.get('protocol', 'ocs')
 
@@ -84,7 +83,7 @@
 
         logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'")
 
-        if os.access(os.path.join(d.getVar('OSCDIR', True), ud.path, ud.module), os.R_OK):
+        if os.access(os.path.join(d.getVar('OSCDIR'), ud.path, ud.module), os.R_OK):
             oscupdatecmd = self._buildosccommand(ud, d, "update")
             logger.info("Update "+ ud.url)
             # update sources there
@@ -112,7 +111,7 @@
         Generate a .oscrc to be used for this run.
         """
 
-        config_path = os.path.join(d.getVar('OSCDIR', True), "oscrc")
+        config_path = os.path.join(d.getVar('OSCDIR'), "oscrc")
         if (os.path.exists(config_path)):
             os.remove(config_path)
 
@@ -121,8 +120,8 @@
         f.write("apisrv = %s\n" % ud.host)
         f.write("scheme = http\n")
         f.write("su-wrapper = su -c\n")
-        f.write("build-root = %s\n" % d.getVar('WORKDIR', True))
-        f.write("urllist = %s\n" % d.getVar("OSCURLLIST", True))
+        f.write("build-root = %s\n" % d.getVar('WORKDIR'))
+        f.write("urllist = %s\n" % d.getVar("OSCURLLIST"))
         f.write("extra-pkgs = gzip\n")
         f.write("\n")
         f.write("[%s]\n" % ud.host)
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/perforce.py b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/perforce.py
index 50cb479..3debad5 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/perforce.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/perforce.py
@@ -26,7 +26,6 @@
 import os
 import logging
 import bb
-from   bb import data
 from   bb.fetch2 import FetchMethod
 from   bb.fetch2 import FetchError
 from   bb.fetch2 import logger
@@ -44,13 +43,13 @@
         provided by the env, use it.  If P4PORT is specified by the recipe, use
         its values, which may override the settings in P4CONFIG.
         """
-        ud.basecmd = d.getVar('FETCHCMD_p4', True)
+        ud.basecmd = d.getVar('FETCHCMD_p4')
         if not ud.basecmd:
             ud.basecmd = "/usr/bin/env p4"
 
-        ud.dldir = d.getVar('P4DIR', True)
+        ud.dldir = d.getVar('P4DIR')
         if not ud.dldir:
-            ud.dldir = '%s/%s' % (d.getVar('DL_DIR', True), 'p4')
+            ud.dldir = '%s/%s' % (d.getVar('DL_DIR'), 'p4')
 
         path = ud.url.split('://')[1]
         path = path.split(';')[0]
@@ -62,7 +61,7 @@
             ud.path = path
 
         ud.usingp4config = False
-        p4port = d.getVar('P4PORT', True)
+        p4port = d.getVar('P4PORT')
 
         if p4port:
             logger.debug(1, 'Using recipe provided P4PORT: %s' % p4port)
@@ -71,7 +70,7 @@
             logger.debug(1, 'Trying to use P4CONFIG to automatically set P4PORT...')
             ud.usingp4config = True
             p4cmd = '%s info | grep "Server address"' % ud.basecmd
-            bb.fetch2.check_network_access(d, p4cmd)
+            bb.fetch2.check_network_access(d, p4cmd, ud.url)
             ud.host = runfetchcmd(p4cmd, d, True)
             ud.host = ud.host.split(': ')[1].strip()
             logger.debug(1, 'Determined P4PORT to be: %s' % ud.host)
@@ -87,9 +86,9 @@
         cleanedhost = ud.host.replace(':', '.')
         ud.pkgdir = os.path.join(ud.dldir, cleanedhost, cleanedpath)
 
-        ud.setup_revisons(d)
+        ud.setup_revisions(d)
 
-        ud.localfile = data.expand('%s_%s_%s.tar.gz' % (cleanedhost, cleanedpath, ud.revision), d)
+        ud.localfile = d.expand('%s_%s_%s.tar.gz' % (cleanedhost, cleanedpath, ud.revision))
 
     def _buildp4command(self, ud, d, command, depot_filename=None):
         """
@@ -140,7 +139,7 @@
         'p4 files' command, including trailing '#rev' file revision indicator
         """
         p4cmd = self._buildp4command(ud, d, 'files')
-        bb.fetch2.check_network_access(d, p4cmd)
+        bb.fetch2.check_network_access(d, p4cmd, ud.url)
         p4fileslist = runfetchcmd(p4cmd, d, True)
         p4fileslist = [f.rstrip() for f in p4fileslist.splitlines()]
 
@@ -171,7 +170,7 @@
 
         for afile in filelist:
             p4fetchcmd = self._buildp4command(ud, d, 'print', afile)
-            bb.fetch2.check_network_access(d, p4fetchcmd)
+            bb.fetch2.check_network_access(d, p4fetchcmd, ud.url)
             runfetchcmd(p4fetchcmd, d, workdir=ud.pkgdir)
 
         runfetchcmd('tar -czf %s p4' % (ud.localpath), d, cleanup=[ud.localpath], workdir=ud.pkgdir)
@@ -191,7 +190,7 @@
     def _latest_revision(self, ud, d, name):
         """ Return the latest upstream scm revision number """
         p4cmd = self._buildp4command(ud, d, "changes")
-        bb.fetch2.check_network_access(d, p4cmd)
+        bb.fetch2.check_network_access(d, p4cmd, ud.url)
         tip = runfetchcmd(p4cmd, d, True)
 
         if not tip:
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/repo.py b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/repo.py
index ecc6e68..1be91cc 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/repo.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/repo.py
@@ -25,7 +25,6 @@
 
 import os
 import bb
-from   bb    import data
 from   bb.fetch2 import FetchMethod
 from   bb.fetch2 import runfetchcmd
 
@@ -51,17 +50,17 @@
         if not ud.manifest.endswith('.xml'):
             ud.manifest += '.xml'
 
-        ud.localfile = data.expand("repo_%s%s_%s_%s.tar.gz" % (ud.host, ud.path.replace("/", "."), ud.manifest, ud.branch), d)
+        ud.localfile = d.expand("repo_%s%s_%s_%s.tar.gz" % (ud.host, ud.path.replace("/", "."), ud.manifest, ud.branch))
 
     def download(self, ud, d):
         """Fetch url"""
 
-        if os.access(os.path.join(data.getVar("DL_DIR", d, True), ud.localfile), os.R_OK):
+        if os.access(os.path.join(d.getVar("DL_DIR"), ud.localfile), os.R_OK):
             logger.debug(1, "%s already exists (or was stashed). Skipping repo init / sync.", ud.localpath)
             return
 
         gitsrcname = "%s%s" % (ud.host, ud.path.replace("/", "."))
-        repodir = data.getVar("REPODIR", d, True) or os.path.join(data.getVar("DL_DIR", d, True), "repo")
+        repodir = d.getVar("REPODIR") or os.path.join(d.getVar("DL_DIR"), "repo")
         codir = os.path.join(repodir, gitsrcname, ud.manifest)
 
         if ud.user:
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/s3.py b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/s3.py
new file mode 100644
index 0000000..1629288
--- /dev/null
+++ b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/s3.py
@@ -0,0 +1,98 @@
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+"""
+BitBake 'Fetch' implementation for Amazon AWS S3.
+
+Class for fetching files from Amazon S3 using the AWS Command Line Interface.
+The aws tool must be correctly installed and configured prior to use.
+
+"""
+
+# Copyright (C) 2017, Andre McCurdy <armccurdy@gmail.com>
+#
+# Based in part on bb.fetch2.wget:
+#    Copyright (C) 2003, 2004  Chris Larson
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Based on functions from the base bb module, Copyright 2003 Holger Schurig
+
+import os
+import bb
+import urllib.request, urllib.parse, urllib.error
+from bb.fetch2 import FetchMethod
+from bb.fetch2 import FetchError
+from bb.fetch2 import runfetchcmd
+
+class S3(FetchMethod):
+    """Class to fetch urls via 'aws s3'"""
+
+    def supports(self, ud, d):
+        """
+        Check to see if a given url can be fetched with s3.
+        """
+        return ud.type in ['s3']
+
+    def recommends_checksum(self, urldata):
+        return True
+
+    def urldata_init(self, ud, d):
+        if 'downloadfilename' in ud.parm:
+            ud.basename = ud.parm['downloadfilename']
+        else:
+            ud.basename = os.path.basename(ud.path)
+
+        ud.localfile = d.expand(urllib.parse.unquote(ud.basename))
+
+        ud.basecmd = d.getVar("FETCHCMD_s3") or "/usr/bin/env aws s3"
+
+    def download(self, ud, d):
+        """
+        Fetch urls
+        Assumes localpath was called first
+        """
+
+        cmd = '%s cp s3://%s%s %s' % (ud.basecmd, ud.host, ud.path, ud.localpath)
+        bb.fetch2.check_network_access(d, cmd, ud.url)
+        runfetchcmd(cmd, d)
+
+        # Additional sanity checks copied from the wget class (although there
+        # are no known issues which mean these are required, treat the aws cli
+        # tool with a little healthy suspicion).
+
+        if not os.path.exists(ud.localpath):
+            raise FetchError("The aws cp command returned success for s3://%s%s but %s doesn't exist?!" % (ud.host, ud.path, ud.localpath))
+
+        if os.path.getsize(ud.localpath) == 0:
+            os.remove(ud.localpath)
+            raise FetchError("The aws cp command for s3://%s%s resulted in a zero size file?! Deleting and failing since this isn't right." % (ud.host, ud.path))
+
+        return True
+
+    def checkstatus(self, fetch, ud, d):
+        """
+        Check the status of a URL
+        """
+
+        cmd = '%s ls s3://%s%s' % (ud.basecmd, ud.host, ud.path)
+        bb.fetch2.check_network_access(d, cmd, ud.url)
+        output = runfetchcmd(cmd, d)
+
+        # "aws s3 ls s3://mybucket/foo" will exit with success even if the file
+        # is not found, so check output of the command to confirm success.
+
+        if not output:
+            raise FetchError("The aws ls command for s3://%s%s gave empty output" % (ud.host, ud.path))
+
+        return True
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/sftp.py b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/sftp.py
index 7989fcc..81884a6 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/sftp.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/sftp.py
@@ -62,12 +62,10 @@
 import os
 import bb
 import urllib.request, urllib.parse, urllib.error
-from bb import data
 from bb.fetch2 import URI
 from bb.fetch2 import FetchMethod
 from bb.fetch2 import runfetchcmd
 
-
 class SFTP(FetchMethod):
     """Class to fetch urls via 'sftp'"""
 
@@ -92,7 +90,7 @@
         else:
             ud.basename = os.path.basename(ud.path)
 
-        ud.localfile = data.expand(urllib.parse.unquote(ud.basename), d)
+        ud.localfile = d.expand(urllib.parse.unquote(ud.basename))
 
     def download(self, ud, d):
         """Fetch urls"""
@@ -104,7 +102,7 @@
             port = '-P %d' % urlo.port
             urlo.port = None
 
-        dldir = data.getVar('DL_DIR', d, True)
+        dldir = d.getVar('DL_DIR')
         lpath = os.path.join(dldir, ud.localfile)
 
         user = ''
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/ssh.py b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/ssh.py
index 56f9b7e..6047ee4 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/ssh.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/ssh.py
@@ -43,7 +43,6 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 import re, os
-from   bb import data
 from   bb.fetch2 import FetchMethod
 from   bb.fetch2 import FetchError
 from   bb.fetch2 import logger
@@ -87,11 +86,11 @@
         m = __pattern__.match(urldata.url)
         path = m.group('path')
         host = m.group('host')
-        urldata.localpath = os.path.join(d.getVar('DL_DIR', True),
+        urldata.localpath = os.path.join(d.getVar('DL_DIR'),
                 os.path.basename(os.path.normpath(path)))
 
     def download(self, urldata, d):
-        dldir = d.getVar('DL_DIR', True)
+        dldir = d.getVar('DL_DIR')
 
         m = __pattern__.match(urldata.url)
         path = m.group('path')
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/svn.py b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/svn.py
index 6ca79d3..3f172ee 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/svn.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/svn.py
@@ -28,7 +28,6 @@
 import logging
 import bb
 import re
-from   bb import data
 from   bb.fetch2 import FetchMethod
 from   bb.fetch2 import FetchError
 from   bb.fetch2 import MissingParameterError
@@ -50,7 +49,7 @@
         if not "module" in ud.parm:
             raise MissingParameterError('module', ud.url)
 
-        ud.basecmd = d.getVar('FETCHCMD_svn', True)
+        ud.basecmd = d.getVar('FETCHCMD_svn')
 
         ud.module = ud.parm["module"]
 
@@ -61,15 +60,15 @@
 
         # Create paths to svn checkouts
         relpath = self._strip_leading_slashes(ud.path)
-        ud.pkgdir = os.path.join(data.expand('${SVNDIR}', d), ud.host, relpath)
+        ud.pkgdir = os.path.join(d.expand('${SVNDIR}'), ud.host, relpath)
         ud.moddir = os.path.join(ud.pkgdir, ud.module)
 
-        ud.setup_revisons(d)
+        ud.setup_revisions(d)
 
         if 'rev' in ud.parm:
             ud.revision = ud.parm['rev']
 
-        ud.localfile = data.expand('%s_%s_%s_%s_.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision), d)
+        ud.localfile = d.expand('%s_%s_%s_%s_.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision))
 
     def _buildsvncommand(self, ud, d, command):
         """
@@ -79,9 +78,9 @@
 
         proto = ud.parm.get('protocol', 'svn')
 
-        svn_rsh = None
-        if proto == "svn+ssh" and "rsh" in ud.parm:
-            svn_rsh = ud.parm["rsh"]
+        svn_ssh = None
+        if proto == "svn+ssh" and "ssh" in ud.parm:
+            svn_ssh = ud.parm["ssh"]
 
         svnroot = ud.host + ud.path
 
@@ -113,8 +112,8 @@
             else:
                 raise FetchError("Invalid svn command %s" % command, ud.url)
 
-        if svn_rsh:
-            svncmd = "svn_RSH=\"%s\" %s" % (svn_rsh, svncmd)
+        if svn_ssh:
+            svncmd = "SVN_SSH=\"%s\" %s" % (svn_ssh, svncmd)
 
         return svncmd
 
@@ -173,7 +172,7 @@
         """
         Return the latest upstream revision number
         """
-        bb.fetch2.check_network_access(d, self._buildsvncommand(ud, d, "log1"))
+        bb.fetch2.check_network_access(d, self._buildsvncommand(ud, d, "log1"), ud.url)
 
         output = runfetchcmd("LANG=C LC_ALL=C " + self._buildsvncommand(ud, d, "log1"), d, True)
 
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/wget.py b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/wget.py
index 23d48ac..ae0ffa8 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/wget.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/wget.py
@@ -33,7 +33,6 @@
 import bb
 import bb.progress
 import urllib.request, urllib.parse, urllib.error
-from   bb import data
 from   bb.fetch2 import FetchMethod
 from   bb.fetch2 import FetchError
 from   bb.fetch2 import logger
@@ -84,18 +83,18 @@
         else:
             ud.basename = os.path.basename(ud.path)
 
-        ud.localfile = data.expand(urllib.parse.unquote(ud.basename), d)
+        ud.localfile = d.expand(urllib.parse.unquote(ud.basename))
         if not ud.localfile:
-            ud.localfile = data.expand(urllib.parse.unquote(ud.host + ud.path).replace("/", "."), d)
+            ud.localfile = d.expand(urllib.parse.unquote(ud.host + ud.path).replace("/", "."))
 
-        self.basecmd = d.getVar("FETCHCMD_wget", True) or "/usr/bin/env wget -t 2 -T 30 --passive-ftp --no-check-certificate"
+        self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -t 2 -T 30 --passive-ftp --no-check-certificate"
 
     def _runwget(self, ud, d, command, quiet):
 
         progresshandler = WgetProgressHandler(d)
 
         logger.debug(2, "Fetching %s using command '%s'" % (ud.url, command))
-        bb.fetch2.check_network_access(d, command)
+        bb.fetch2.check_network_access(d, command, ud.url)
         runfetchcmd(command + ' --progress=dot -v', d, quiet, log=progresshandler)
 
     def download(self, ud, d):
@@ -104,7 +103,7 @@
         fetchcmd = self.basecmd
 
         if 'downloadfilename' in ud.parm:
-            dldir = d.getVar("DL_DIR", True)
+            dldir = d.getVar("DL_DIR")
             bb.utils.mkdirhier(os.path.dirname(dldir + os.sep + ud.localfile))
             fetchcmd += " -O " + dldir + os.sep + ud.localfile
 
@@ -304,12 +303,24 @@
             r = urllib.request.Request(uri)
             r.get_method = lambda: "HEAD"
 
-            if ud.user:
+            def add_basic_auth(login_str, request):
+                '''Adds Basic auth to http request, pass in login:password as string'''
                 import base64
-                encodeuser = base64.b64encode(ud.user.encode('utf-8')).decode("utf-8")
+                encodeuser = base64.b64encode(login_str.encode('utf-8')).decode("utf-8")
                 authheader =  "Basic %s" % encodeuser
                 r.add_header("Authorization", authheader)
 
+            if ud.user:
+                add_basic_auth(ud.user, r)
+
+            try:
+                import netrc, urllib.parse
+                n = netrc.netrc()
+                login, unused, password = n.authenticators(urllib.parse.urlparse(uri).hostname)
+                add_basic_auth("%s:%s" % (login, password), r)
+            except (TypeError, ImportError, IOError, netrc.NetrcParseError):
+                 pass
+
             opener.open(r)
         except urllib.error.URLError as e:
             if try_again:
@@ -534,7 +545,7 @@
 
         # src.rpm extension was added only for rpm package. Can be removed if the rpm
         # packaged will always be considered as having to be manually upgraded
-        psuffix_regex = "(tar\.gz|tgz|tar\.bz2|zip|xz|rpm|bz2|orig\.tar\.gz|tar\.xz|src\.tar\.gz|src\.tgz|svnr\d+\.tar\.bz2|stable\.tar\.gz|src\.rpm)"
+        psuffix_regex = "(tar\.gz|tgz|tar\.bz2|zip|xz|tar\.lz|rpm|bz2|orig\.tar\.gz|tar\.xz|src\.tar\.gz|src\.tgz|svnr\d+\.tar\.bz2|stable\.tar\.gz|src\.rpm)"
 
         # match name, version and archive type of a package
         package_regex_comp = re.compile("(?P<name>%s?\.?v?)(?P<pver>%s)(?P<arch>%s)?[\.-](?P<type>%s$)"
@@ -542,7 +553,7 @@
         self.suffix_regex_comp = re.compile(psuffix_regex)
 
         # compile regex, can be specific by package or generic regex
-        pn_regex = d.getVar('UPSTREAM_CHECK_REGEX', True)
+        pn_regex = d.getVar('UPSTREAM_CHECK_REGEX')
         if pn_regex:
             package_custom_regex_comp = re.compile(pn_regex)
         else:
@@ -563,7 +574,7 @@
         sanity check to ensure same name and type.
         """
         package = ud.path.split("/")[-1]
-        current_version = ['', d.getVar('PV', True), '']
+        current_version = ['', d.getVar('PV'), '']
 
         """possible to have no version in pkg name, such as spectrum-fw"""
         if not re.search("\d+", package):
@@ -578,7 +589,7 @@
         bb.debug(3, "latest_versionstring, regex: %s" % (package_regex.pattern))
 
         uri = ""
-        regex_uri = d.getVar("UPSTREAM_CHECK_URI", True)
+        regex_uri = d.getVar("UPSTREAM_CHECK_URI")
         if not regex_uri:
             path = ud.path.split(package)[0]
 
@@ -587,7 +598,7 @@
             dirver_regex = re.compile("(?P<dirver>[^/]*(\d+\.)*\d+([-_]r\d+)*)/")
             m = dirver_regex.search(path)
             if m:
-                pn = d.getVar('PN', True)
+                pn = d.getVar('PN')
                 dirver = m.group('dirver')
 
                 dirver_pn_regex = re.compile("%s\d?" % (re.escape(pn)))
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/main.py b/import-layers/yocto-poky/bitbake/lib/bb/main.py
index f2f59f6..8c948c2 100755
--- a/import-layers/yocto-poky/bitbake/lib/bb/main.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/main.py
@@ -174,13 +174,24 @@
                           help="Read the specified file after bitbake.conf.")
 
         parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False,
-                          help="Output more log message data to the terminal.")
+                          help="Enable tracing of shell tasks (with 'set -x'). "
+                               "Also print bb.note(...) messages to stdout (in "
+                               "addition to writing them to ${T}/log.do_<task>).")
 
         parser.add_option("-D", "--debug", action="count", dest="debug", default=0,
-                          help="Increase the debug level. You can specify this more than once.")
+                          help="Increase the debug level. You can specify this "
+                               "more than once. -D sets the debug level to 1, "
+                               "where only bb.debug(1, ...) messages are printed "
+                               "to stdout; -DD sets the debug level to 2, where "
+                               "both bb.debug(1, ...) and bb.debug(2, ...) "
+                               "messages are printed; etc. Without -D, no debug "
+                               "messages are printed. Note that -D only affects "
+                               "output to stdout. All debug messages are written "
+                               "to ${T}/log.do_taskname, regardless of the debug "
+                               "level.")
 
-        parser.add_option("-q", "--quiet", action="store_true", dest="quiet", default=False,
-                          help="Output less log message data to the terminal.")
+        parser.add_option("-q", "--quiet", action="count", dest="quiet", default=0,
+                          help="Output less log message data to the terminal. You can specify this more than once.")
 
         parser.add_option("-n", "--dry-run", action="store_true", dest="dry_run", default=False,
                           help="Don't execute, just go through the motions.")
@@ -287,6 +298,9 @@
                           help="Writes the event log of the build to a bitbake event json file. "
                                "Use '' (empty string) to assign the name automatically.")
 
+        parser.add_option("", "--runall", action="store", dest="runall",
+                          help="Run the specified task for all build targets and their dependencies.")
+
         options, targets = parser.parse_args(argv)
 
         if options.quiet and options.verbose:
@@ -367,6 +381,7 @@
         raise
     if not configParams.foreground:
         server.detach()
+        cooker.shutdown()
     cooker.lock.close()
     return server
 
@@ -389,12 +404,8 @@
     except:
         pass
 
-
     configuration.setConfigParameters(configParams)
 
-    ui_module = import_extension_module(bb.ui, configParams.ui, 'main')
-    servermodule = import_extension_module(bb.server, configParams.servertype, 'BitBakeServer')
-
     if configParams.server_only:
         if configParams.servertype != "xmlrpc":
             raise BBMainException("FATAL: If '--server-only' is defined, we must set the "
@@ -442,66 +453,11 @@
     bb.msg.init_msgconfig(configParams.verbose, configuration.debug,
                           configuration.debug_domains)
 
-    # Ensure logging messages get sent to the UI as events
-    handler = bb.event.LogHandler()
-    if not configParams.status_only:
-        # In status only mode there are no logs and no UI
-        logger.addHandler(handler)
-
-    # Clear away any spurious environment variables while we stoke up the cooker
-    cleanedvars = bb.utils.clean_environment()
-
-    featureset = []
-    if not configParams.server_only:
-        # Collect the feature set for the UI
-        featureset = getattr(ui_module, "featureSet", [])
-
-    if configParams.server_only:
-        for param in ('prefile', 'postfile'):
-            value = getattr(configParams, param)
-            if value:
-                setattr(configuration, "%s_server" % param, value)
-                param = "%s_server" % param
-
-    if not configParams.remote_server:
-        # we start a server with a given configuration
-        server = start_server(servermodule, configParams, configuration, featureset)
-        bb.event.ui_queue = []
-    else:
-        if os.getenv('BBSERVER') == 'autostart':
-            if configParams.remote_server == 'autostart' or \
-               not servermodule.check_connection(configParams.remote_server, timeout=2):
-                configParams.bind = 'localhost:0'
-                srv = start_server(servermodule, configParams, configuration, featureset)
-                configParams.remote_server = '%s:%d' % tuple(configuration.interface)
-                bb.event.ui_queue = []
-
-        # we start a stub server that is actually a XMLRPClient that connects to a real server
-        server = servermodule.BitBakeXMLRPCClient(configParams.observe_only,
-                                                  configParams.xmlrpctoken)
-        server.saveConnectionDetails(configParams.remote_server)
-
+    server, server_connection, ui_module = setup_bitbake(configParams, configuration)
+    if server_connection is None and configParams.kill_server:
+        return 0
 
     if not configParams.server_only:
-        try:
-            server_connection = server.establishConnection(featureset)
-        except Exception as e:
-            bb.fatal("Could not connect to server %s: %s" % (configParams.remote_server, str(e)))
-
-        if configParams.kill_server:
-            server_connection.connection.terminateServer()
-            bb.event.ui_queue = []
-            return 0
-
-        server_connection.setupEventQueue()
-
-        # Restore the environment in case the UI needs it
-        for k in cleanedvars:
-            os.environ[k] = cleanedvars[k]
-
-        logger.removeHandler(handler)
-
-
         if configParams.status_only:
             server_connection.terminate()
             return 0
@@ -520,3 +476,77 @@
         return 0
 
     return 1
+
+def setup_bitbake(configParams, configuration, extrafeatures=None):
+    # Ensure logging messages get sent to the UI as events
+    handler = bb.event.LogHandler()
+    if not configParams.status_only:
+        # In status only mode there are no logs and no UI
+        logger.addHandler(handler)
+
+    # Clear away any spurious environment variables while we stoke up the cooker
+    cleanedvars = bb.utils.clean_environment()
+
+    if configParams.server_only:
+        featureset = []
+        ui_module = None
+    else:
+        ui_module = import_extension_module(bb.ui, configParams.ui, 'main')
+        # Collect the feature set for the UI
+        featureset = getattr(ui_module, "featureSet", [])
+
+    if configParams.server_only:
+        for param in ('prefile', 'postfile'):
+            value = getattr(configParams, param)
+            if value:
+                setattr(configuration, "%s_server" % param, value)
+                param = "%s_server" % param
+
+    if extrafeatures:
+        for feature in extrafeatures:
+            if not feature in featureset:
+                featureset.append(feature)
+
+    servermodule = import_extension_module(bb.server,
+                                            configParams.servertype,
+                                            'BitBakeServer')
+    if configParams.remote_server:
+        if os.getenv('BBSERVER') == 'autostart':
+            if configParams.remote_server == 'autostart' or \
+               not servermodule.check_connection(configParams.remote_server, timeout=2):
+                configParams.bind = 'localhost:0'
+                srv = start_server(servermodule, configParams, configuration, featureset)
+                configParams.remote_server = '%s:%d' % tuple(configuration.interface)
+                bb.event.ui_queue = []
+        # we start a stub server that is actually a XMLRPClient that connects to a real server
+        from bb.server.xmlrpc import BitBakeXMLRPCClient
+        server = servermodule.BitBakeXMLRPCClient(configParams.observe_only,
+                                                  configParams.xmlrpctoken)
+        server.saveConnectionDetails(configParams.remote_server)
+    else:
+        # we start a server with a given configuration
+        server = start_server(servermodule, configParams, configuration, featureset)
+        bb.event.ui_queue = []
+
+    if configParams.server_only:
+        server_connection = None
+    else:
+        try:
+            server_connection = server.establishConnection(featureset)
+        except Exception as e:
+            bb.fatal("Could not connect to server %s: %s" % (configParams.remote_server, str(e)))
+
+        if configParams.kill_server:
+            server_connection.connection.terminateServer()
+            bb.event.ui_queue = []
+            return None, None, None
+
+        server_connection.setupEventQueue()
+
+        # Restore the environment in case the UI needs it
+        for k in cleanedvars:
+            os.environ[k] = cleanedvars[k]
+
+        logger.removeHandler(handler)
+
+    return server, server_connection, ui_module
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/monitordisk.py b/import-layers/yocto-poky/bitbake/lib/bb/monitordisk.py
index 203c405..833cd3d 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/monitordisk.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/monitordisk.py
@@ -129,7 +129,7 @@
             bb.utils.mkdirhier(path)
         dev = getMountedDev(path)
         # Use path/action as the key
-        devDict[os.path.join(path, action)] = [dev, minSpace, minInode]
+        devDict[(path, action)] = [dev, minSpace, minInode]
 
     return devDict
 
@@ -141,7 +141,7 @@
     spaceDefault = 50 * 1024 * 1024
     inodeDefault = 5 * 1024
 
-    interval = configuration.getVar("BB_DISKMON_WARNINTERVAL", True)
+    interval = configuration.getVar("BB_DISKMON_WARNINTERVAL")
     if not interval:
         return spaceDefault, inodeDefault
     else:
@@ -179,7 +179,7 @@
         self.enableMonitor = False
         self.configuration = configuration
 
-        BBDirs = configuration.getVar("BB_DISKMON_DIRS", True) or None
+        BBDirs = configuration.getVar("BB_DISKMON_DIRS") or None
         if BBDirs:
             self.devDict = getDiskData(BBDirs, configuration)
             if self.devDict:
@@ -205,18 +205,21 @@
         """ Take action for the monitor """
 
         if self.enableMonitor:
-            for k in self.devDict:
-                path = os.path.dirname(k)
-                action = os.path.basename(k)
-                dev = self.devDict[k][0]
-                minSpace = self.devDict[k][1]
-                minInode = self.devDict[k][2]
+            diskUsage = {}
+            for k, attributes in self.devDict.items():
+                path, action = k
+                dev, minSpace, minInode = attributes
 
                 st = os.statvfs(path)
 
-                # The free space, float point number
+                # The available free space, integer number
                 freeSpace = st.f_bavail * st.f_frsize
 
+                # Send all relevant information in the event.
+                freeSpaceRoot = st.f_bfree * st.f_frsize
+                totalSpace = st.f_blocks * st.f_frsize
+                diskUsage[dev] = bb.event.DiskUsageSample(freeSpace, freeSpaceRoot, totalSpace)
+
                 if minSpace and freeSpace < minSpace:
                     # Always show warning, the self.checked would always be False if the action is WARN
                     if self.preFreeS[k] == 0 or self.preFreeS[k] - freeSpace > self.spaceInterval and not self.checked[k]:
@@ -235,7 +238,7 @@
                         rq.finish_runqueue(True)
                         bb.event.fire(bb.event.DiskFull(dev, 'disk', freeSpace, path), self.configuration)
 
-                # The free inodes, float point number
+                # The free inodes, integer number
                 freeInode = st.f_favail
 
                 if minInode and freeInode < minInode:
@@ -260,4 +263,6 @@
                         self.checked[k] = True
                         rq.finish_runqueue(True)
                         bb.event.fire(bb.event.DiskFull(dev, 'inode', freeInode, path), self.configuration)
+
+            bb.event.fire(bb.event.MonitorDiskEvent(diskUsage), self.configuration)
         return
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/msg.py b/import-layers/yocto-poky/bitbake/lib/bb/msg.py
index b7c39fa..90b1582 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/msg.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/msg.py
@@ -201,3 +201,18 @@
         logger = logging.getLogger("BitBake")
     logger.critical(msg)
     sys.exit(1)
+
+def logger_create(name, output=sys.stderr, level=logging.INFO, preserve_handlers=False, color='auto'):
+    """Standalone logger creation function"""
+    logger = logging.getLogger(name)
+    console = logging.StreamHandler(output)
+    format = bb.msg.BBLogFormatter("%(levelname)s: %(message)s")
+    if color == 'always' or (color == 'auto' and output.isatty()):
+        format.enable_color()
+    console.setFormatter(format)
+    if preserve_handlers:
+        logger.addHandler(console)
+    else:
+        logger.handlers = [console]
+    logger.setLevel(level)
+    return logger
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/parse/__init__.py b/import-layers/yocto-poky/bitbake/lib/bb/parse/__init__.py
index 26ae7ea..a2952ec 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/parse/__init__.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/parse/__init__.py
@@ -123,7 +123,7 @@
 
 def resolve_file(fn, d):
     if not os.path.isabs(fn):
-        bbpath = d.getVar("BBPATH", True)
+        bbpath = d.getVar("BBPATH")
         newfn, attempts = bb.utils.which(bbpath, fn, history=True)
         for af in attempts:
             mark_dependency(d, af)
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/parse/ast.py b/import-layers/yocto-poky/bitbake/lib/bb/parse/ast.py
index fa83b18..dba4540 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/parse/ast.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/parse/ast.py
@@ -30,8 +30,6 @@
 from bb import methodpool
 from bb.parse import logger
 
-_bbversions_re = re.compile(r"\[(?P<from>[0-9]+)-(?P<to>[0-9]+)\]")
-
 class StatementGroup(list):
     def eval(self, data):
         for statement in self:
@@ -132,7 +130,6 @@
                 val = groupd["value"]
         elif "colon" in groupd and groupd["colon"] != None:
             e = data.createCopy()
-            bb.data.update_data(e)
             op = "immediate"
             val = e.expand(groupd["value"], key + "[:=]")
         elif "append" in groupd and groupd["append"] != None:
@@ -347,19 +344,18 @@
         if not handlerfn:
             bb.fatal("Undefined event handler function '%s'" % var)
         handlerln = int(d.getVarFlag(var, "lineno", False))
-        bb.event.register(var, d.getVar(var, False), (d.getVarFlag(var, "eventmask", True) or "").split(), handlerfn, handlerln)
+        bb.event.register(var, d.getVar(var, False), (d.getVarFlag(var, "eventmask") or "").split(), handlerfn, handlerln)
 
     bb.event.fire(bb.event.RecipePreFinalise(fn), d)
 
     bb.data.expandKeys(d)
-    bb.data.update_data(d)
     code = []
     for funcname in d.getVar("__BBANONFUNCS", False) or []:
         code.append("%s(d)" % funcname)
     bb.utils.better_exec("\n".join(code), {"d": d})
-    bb.data.update_data(d)
 
     tasklist = d.getVar('__BBTASKS', False) or []
+    bb.event.fire(bb.event.RecipeTaskPreProcess(fn, list(tasklist)), d)
     bb.build.add_tasks(tasklist, d)
 
     bb.parse.siggen.finalise(fn, d, variant)
@@ -385,29 +381,8 @@
             else:
                 create_variant("%s-%s" % (variant, name), datastores[variant], name)
 
-def _expand_versions(versions):
-    def expand_one(version, start, end):
-        for i in range(start, end + 1):
-            ver = _bbversions_re.sub(str(i), version, 1)
-            yield ver
-
-    versions = iter(versions)
-    while True:
-        try:
-            version = next(versions)
-        except StopIteration:
-            break
-
-        range_ver = _bbversions_re.search(version)
-        if not range_ver:
-            yield version
-        else:
-            newversions = expand_one(version, int(range_ver.group("from")),
-                                     int(range_ver.group("to")))
-            versions = itertools.chain(newversions, versions)
-
 def multi_finalize(fn, d):
-    appends = (d.getVar("__BBAPPEND", True) or "").split()
+    appends = (d.getVar("__BBAPPEND") or "").split()
     for append in appends:
         logger.debug(1, "Appending .bbappend file %s to %s", append, fn)
         bb.parse.BBHandler.handle(append, d, True)
@@ -422,51 +397,7 @@
         d.setVar("__SKIPPED", e.args[0])
     datastores = {"": safe_d}
 
-    versions = (d.getVar("BBVERSIONS", True) or "").split()
-    if versions:
-        pv = orig_pv = d.getVar("PV", True)
-        baseversions = {}
-
-        def verfunc(ver, d, pv_d = None):
-            if pv_d is None:
-                pv_d = d
-
-            overrides = d.getVar("OVERRIDES", True).split(":")
-            pv_d.setVar("PV", ver)
-            overrides.append(ver)
-            bpv = baseversions.get(ver) or orig_pv
-            pv_d.setVar("BPV", bpv)
-            overrides.append(bpv)
-            d.setVar("OVERRIDES", ":".join(overrides))
-
-        versions = list(_expand_versions(versions))
-        for pos, version in enumerate(list(versions)):
-            try:
-                pv, bpv = version.split(":", 2)
-            except ValueError:
-                pass
-            else:
-                versions[pos] = pv
-                baseversions[pv] = bpv
-
-        if pv in versions and not baseversions.get(pv):
-            versions.remove(pv)
-        else:
-            pv = versions.pop()
-
-            # This is necessary because our existing main datastore
-            # has already been finalized with the old PV, we need one
-            # that's been finalized with the new PV.
-            d = bb.data.createCopy(safe_d)
-            verfunc(pv, d, safe_d)
-            try:
-                finalize(fn, d)
-            except bb.parse.SkipRecipe as e:
-                d.setVar("__SKIPPED", e.args[0])
-
-        _create_variants(datastores, versions, verfunc, onlyfinalise)
-
-    extended = d.getVar("BBCLASSEXTEND", True) or ""
+    extended = d.getVar("BBCLASSEXTEND") or ""
     if extended:
         # the following is to support bbextends with arguments, for e.g. multilib
         # an example is as follows:
@@ -484,7 +415,7 @@
             else:
                 extendedmap[ext] = ext
 
-        pn = d.getVar("PN", True)
+        pn = d.getVar("PN")
         def extendfunc(name, d):
             if name != extendedmap[name]:
                 d.setVar("BBEXTENDCURR", extendedmap[name])
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/parse/parse_py/BBHandler.py b/import-layers/yocto-poky/bitbake/lib/bb/parse/parse_py/BBHandler.py
index c54a079..fe918a4 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/parse/parse_py/BBHandler.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/parse/parse_py/BBHandler.py
@@ -66,7 +66,7 @@
             file = os.path.join('classes', '%s.bbclass' % file)
 
         if not os.path.isabs(file):
-            bbpath = d.getVar("BBPATH", True)
+            bbpath = d.getVar("BBPATH")
             abs_fn, attempts = bb.utils.which(bbpath, file, history=True)
             for af in attempts:
                 if af != abs_fn:
@@ -87,17 +87,17 @@
     try:
         return cached_statements[absolute_filename]
     except KeyError:
-        file = open(absolute_filename, 'r')
-        statements = ast.StatementGroup()
+        with open(absolute_filename, 'r') as f:
+            statements = ast.StatementGroup()
 
-        lineno = 0
-        while True:
-            lineno = lineno + 1
-            s = file.readline()
-            if not s: break
-            s = s.rstrip()
-            feeder(lineno, s, filename, base_name, statements)
-        file.close()
+            lineno = 0
+            while True:
+                lineno = lineno + 1
+                s = f.readline()
+                if not s: break
+                s = s.rstrip()
+                feeder(lineno, s, filename, base_name, statements)
+
         if __inpython__:
             # add a blank line to close out any python definition
             feeder(lineno, "", filename, base_name, statements, eof=True)
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/parse/parse_py/ConfHandler.py b/import-layers/yocto-poky/bitbake/lib/bb/parse/parse_py/ConfHandler.py
index 875250d..f7d0cf7 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/parse/parse_py/ConfHandler.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/parse/parse_py/ConfHandler.py
@@ -33,7 +33,7 @@
 __config_regexp__  = re.compile( r"""
     ^
     (?P<exp>export\s*)?
-    (?P<var>[a-zA-Z0-9\-~_+.${}/]+?)
+    (?P<var>[a-zA-Z0-9\-_+.${}/~]+?)
     (\[(?P<flag>[a-zA-Z0-9\-_+.]+)\])?
 
     \s* (
@@ -56,9 +56,9 @@
     """, re.X)
 __include_regexp__ = re.compile( r"include\s+(.+)" )
 __require_regexp__ = re.compile( r"require\s+(.+)" )
-__export_regexp__ = re.compile( r"export\s+([a-zA-Z0-9\-_+.${}/]+)$" )
-__unset_regexp__ = re.compile( r"unset\s+([a-zA-Z0-9\-_+.${}/]+)$" )
-__unset_flag_regexp__ = re.compile( r"unset\s+([a-zA-Z0-9\-_+.${}/]+)\[([a-zA-Z0-9\-_+.${}/]+)\]$" )
+__export_regexp__ = re.compile( r"export\s+([a-zA-Z0-9\-_+.${}/~]+)$" )
+__unset_regexp__ = re.compile( r"unset\s+([a-zA-Z0-9\-_+.${}/~]+)$" )
+__unset_flag_regexp__ = re.compile( r"unset\s+([a-zA-Z0-9\-_+.${}/~]+)\[([a-zA-Z0-9\-_+.]+)\]$" )
 
 def init(data):
     topdir = data.getVar('TOPDIR', False)
@@ -83,16 +83,16 @@
 
     if not os.path.isabs(fn):
         dname = os.path.dirname(parentfn)
-        bbpath = "%s:%s" % (dname, data.getVar("BBPATH", True))
+        bbpath = "%s:%s" % (dname, data.getVar("BBPATH"))
         abs_fn, attempts = bb.utils.which(bbpath, fn, history=True)
         if abs_fn and bb.parse.check_dependency(data, abs_fn):
-            logger.warning("Duplicate inclusion for %s in %s" % (abs_fn, data.getVar('FILE', True)))
+            logger.warning("Duplicate inclusion for %s in %s" % (abs_fn, data.getVar('FILE')))
         for af in attempts:
             bb.parse.mark_dependency(data, af)
         if abs_fn:
             fn = abs_fn
     elif bb.parse.check_dependency(data, fn):
-        logger.warning("Duplicate inclusion for %s in %s" % (fn, data.getVar('FILE', True)))
+        logger.warning("Duplicate inclusion for %s in %s" % (fn, data.getVar('FILE')))
 
     try:
         bb.parse.handle(fn, data, True)
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/persist_data.py b/import-layers/yocto-poky/bitbake/lib/bb/persist_data.py
index bb6deca..bef7018 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/persist_data.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/persist_data.py
@@ -28,11 +28,7 @@
 import warnings
 from bb.compat import total_ordering
 from collections import Mapping
-
-try:
-    import sqlite3
-except ImportError:
-    from pysqlite2 import dbapi2 as sqlite3
+import sqlite3
 
 sqlversion = sqlite3.sqlite_version_info
 if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
@@ -207,8 +203,8 @@
 def persist(domain, d):
     """Convenience factory for SQLTable objects based upon metadata"""
     import bb.utils
-    cachedir = (d.getVar("PERSISTENT_DIR", True) or
-                d.getVar("CACHE", True))
+    cachedir = (d.getVar("PERSISTENT_DIR") or
+                d.getVar("CACHE"))
     if not cachedir:
         logger.critical("Please set the 'PERSISTENT_DIR' or 'CACHE' variable")
         sys.exit(1)
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/process.py b/import-layers/yocto-poky/bitbake/lib/bb/process.py
index c62d7bc..a4a5599 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/process.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/process.py
@@ -162,9 +162,9 @@
         stdout, stderr = _logged_communicate(pipe, log, input, extrafiles)
     else:
         stdout, stderr = pipe.communicate(input)
-        if stdout:
+        if not stdout is None:
             stdout = stdout.decode("utf-8")
-        if stderr:
+        if not stderr is None:
             stderr = stderr.decode("utf-8")
 
     if pipe.returncode != 0:
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/providers.py b/import-layers/yocto-poky/bitbake/lib/bb/providers.py
index db02a0b..443187e 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/providers.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/providers.py
@@ -48,7 +48,6 @@
 
     # Need to ensure data store is expanded
     localdata = data.createCopy(cfgData)
-    bb.data.update_data(localdata)
     bb.data.expandKeys(localdata)
 
     preferred_versions = {}
@@ -123,11 +122,11 @@
 
     # pn can contain '_', e.g. gcc-cross-x86_64 and an override cannot
     # hence we do this manually rather than use OVERRIDES
-    preferred_v = cfgData.getVar("PREFERRED_VERSION_pn-%s" % pn, True)
+    preferred_v = cfgData.getVar("PREFERRED_VERSION_pn-%s" % pn)
     if not preferred_v:
-        preferred_v = cfgData.getVar("PREFERRED_VERSION_%s" % pn, True)
+        preferred_v = cfgData.getVar("PREFERRED_VERSION_%s" % pn)
     if not preferred_v:
-        preferred_v = cfgData.getVar("PREFERRED_VERSION", True)
+        preferred_v = cfgData.getVar("PREFERRED_VERSION")
 
     if preferred_v:
         m = re.match('(\d+:)*(.*)(_.*)*', preferred_v)
@@ -289,7 +288,7 @@
 
     eligible = _filterProviders(providers, item, cfgData, dataCache)
 
-    prefervar = cfgData.getVar('PREFERRED_PROVIDER_%s' % item, True)
+    prefervar = cfgData.getVar('PREFERRED_PROVIDER_%s' % item)
     if prefervar:
         dataCache.preferred[item] = prefervar
 
@@ -318,7 +317,7 @@
     eligible = _filterProviders(providers, item, cfgData, dataCache)
 
     # First try and match any PREFERRED_RPROVIDER entry
-    prefervar = cfgData.getVar('PREFERRED_RPROVIDER_%s' % item, True)
+    prefervar = cfgData.getVar('PREFERRED_RPROVIDER_%s' % item)
     foundUnique = False
     if prefervar:
         for p in eligible:
@@ -345,7 +344,7 @@
             pn = dataCache.pkg_fn[p]
             provides = dataCache.pn_provides[pn]
             for provide in provides:
-                prefervar = cfgData.getVar('PREFERRED_PROVIDER_%s' % provide, True)
+                prefervar = cfgData.getVar('PREFERRED_PROVIDER_%s' % provide)
                 #logger.debug(1, "checking PREFERRED_PROVIDER_%s (value %s) against %s", provide, prefervar, pns.keys())
                 if prefervar in pns and pns[prefervar] not in preferred:
                     var = "PREFERRED_PROVIDER_%s = %s" % (provide, prefervar)
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/remotedata.py b/import-layers/yocto-poky/bitbake/lib/bb/remotedata.py
new file mode 100644
index 0000000..68ecffc
--- /dev/null
+++ b/import-layers/yocto-poky/bitbake/lib/bb/remotedata.py
@@ -0,0 +1,116 @@
+"""
+BitBake 'remotedata' module
+
+Provides support for using a datastore from the bitbake client
+"""
+
+# Copyright (C) 2016  Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import bb.data
+
+class RemoteDatastores:
+    """Used on the server side to manage references to server-side datastores"""
+    def __init__(self, cooker):
+        self.cooker = cooker
+        self.datastores = {}
+        self.locked = []
+        self.nextindex = 1
+
+    def __len__(self):
+        return len(self.datastores)
+
+    def __getitem__(self, key):
+        if key is None:
+            return self.cooker.data
+        else:
+            return self.datastores[key]
+
+    def items(self):
+        return self.datastores.items()
+
+    def store(self, d, locked=False):
+        """
+        Put a datastore into the collection. If locked=True then the datastore
+        is understood to be managed externally and cannot be released by calling
+        release().
+        """
+        idx = self.nextindex
+        self.datastores[idx] = d
+        if locked:
+            self.locked.append(idx)
+        self.nextindex += 1
+        return idx
+
+    def check_store(self, d, locked=False):
+        """
+        Put a datastore into the collection if it's not already in there;
+        in either case return the index
+        """
+        for key, val in self.datastores.items():
+            if val is d:
+                idx = key
+                break
+        else:
+            idx = self.store(d, locked)
+        return idx
+
+    def release(self, idx):
+        """Discard a datastore in the collection"""
+        if idx in self.locked:
+            raise Exception('Tried to release locked datastore %d' % idx)
+        del self.datastores[idx]
+
+    def receive_datastore(self, remote_data):
+        """Receive a datastore object sent from the client (as prepared by transmit_datastore())"""
+        dct = dict(remote_data)
+        d = bb.data_smart.DataSmart()
+        d.dict = dct
+        while True:
+            if '_remote_data' in dct:
+                dsindex = dct['_remote_data']['_content']
+                del dct['_remote_data']
+                if dsindex is None:
+                    dct['_data'] = self.cooker.data.dict
+                else:
+                    dct['_data'] = self.datastores[dsindex].dict
+                break
+            elif '_data' in dct:
+                idct = dict(dct['_data'])
+                dct['_data'] = idct
+                dct = idct
+            else:
+                break
+        return d
+
+    @staticmethod
+    def transmit_datastore(d):
+        """Prepare a datastore object for sending over IPC from the client end"""
+        # FIXME content might be a dict, need to turn that into a list as well
+        def copy_dicts(dct):
+            if '_remote_data' in dct:
+                dsindex = dct['_remote_data']['_content'].dsindex
+                newdct = dct.copy()
+                newdct['_remote_data'] = {'_content': dsindex}
+                return list(newdct.items())
+            elif '_data' in dct:
+                newdct = dct.copy()
+                newdata = copy_dicts(dct['_data'])
+                if newdata:
+                    newdct['_data'] = newdata
+                return list(newdct.items())
+            return None
+        main_dict = copy_dicts(d.dict)
+        return main_dict
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/runqueue.py b/import-layers/yocto-poky/bitbake/lib/bb/runqueue.py
index 9384c72..7d2ff81 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/runqueue.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/runqueue.py
@@ -36,6 +36,7 @@
 from bb import monitordisk
 import subprocess
 import pickle
+from multiprocessing import Process
 
 bblogger = logging.getLogger("BitBake")
 logger = logging.getLogger("BitBake.RunQueue")
@@ -183,6 +184,18 @@
     def newbuilable(self, task):
         self.buildable.append(task)
 
+    def describe_task(self, taskid):
+        result = 'ID %s' % taskid
+        if self.rev_prio_map:
+            result = result + (' pri %d' % self.rev_prio_map[taskid])
+        return result
+
+    def dump_prio(self, comment):
+        bb.debug(3, '%s (most important first):\n%s' %
+                 (comment,
+                  '\n'.join(['%d. %s' % (index + 1, self.describe_task(taskid)) for
+                             index, taskid in enumerate(self.prio_map)])))
+
 class RunQueueSchedulerSpeed(RunQueueScheduler):
     """
     A scheduler optimised for speed. The priority map is sorted by task weight,
@@ -212,35 +225,100 @@
 
 class RunQueueSchedulerCompletion(RunQueueSchedulerSpeed):
     """
-    A scheduler optimised to complete .bb files are quickly as possible. The
+    A scheduler optimised to complete .bb files as quickly as possible. The
     priority map is sorted by task weight, but then reordered so once a given
-    .bb file starts to build, it's completed as quickly as possible. This works
-    well where disk space is at a premium and classes like OE's rm_work are in
-    force.
+    .bb file starts to build, it's completed as quickly as possible by
+    running all tasks related to the same .bb file one after the after.
+    This works well where disk space is at a premium and classes like OE's
+    rm_work are in force.
     """
     name = "completion"
 
     def __init__(self, runqueue, rqdata):
-        RunQueueSchedulerSpeed.__init__(self, runqueue, rqdata)
+        super(RunQueueSchedulerCompletion, self).__init__(runqueue, rqdata)
 
-        #FIXME - whilst this groups all fns together it does not reorder the
-        #fn groups optimally.
+        # Extract list of tasks for each recipe, with tasks sorted
+        # ascending from "must run first" (typically do_fetch) to
+        # "runs last" (do_build). The speed scheduler prioritizes
+        # tasks that must run first before the ones that run later;
+        # this is what we depend on here.
+        task_lists = {}
+        for taskid in self.prio_map:
+            fn, taskname = taskid.rsplit(':', 1)
+            task_lists.setdefault(fn, []).append(taskname)
 
-        basemap = copy.deepcopy(self.prio_map)
-        self.prio_map = []
-        while (len(basemap) > 0):
-            entry = basemap.pop(0)
-            self.prio_map.append(entry)
-            fn = fn_from_tid(entry)
-            todel = []
-            for entry in basemap:
-                entry_fn = fn_from_tid(entry)
-                if entry_fn == fn:
-                    todel.append(basemap.index(entry))
-                    self.prio_map.append(entry)
-            todel.reverse()
-            for idx in todel:
-                del basemap[idx]
+        # Now unify the different task lists. The strategy is that
+        # common tasks get skipped and new ones get inserted after the
+        # preceeding common one(s) as they are found. Because task
+        # lists should differ only by their number of tasks, but not
+        # the ordering of the common tasks, this should result in a
+        # deterministic result that is a superset of the individual
+        # task ordering.
+        all_tasks = []
+        for recipe, new_tasks in task_lists.items():
+            index = 0
+            old_task = all_tasks[index] if index < len(all_tasks) else None
+            for new_task in new_tasks:
+                if old_task == new_task:
+                    # Common task, skip it. This is the fast-path which
+                    # avoids a full search.
+                    index += 1
+                    old_task = all_tasks[index] if index < len(all_tasks) else None
+                else:
+                    try:
+                        index = all_tasks.index(new_task)
+                        # Already present, just not at the current
+                        # place. We re-synchronized by changing the
+                        # index so that it matches again. Now
+                        # move on to the next existing task.
+                        index += 1
+                        old_task = all_tasks[index] if index < len(all_tasks) else None
+                    except ValueError:
+                        # Not present. Insert before old_task, which
+                        # remains the same (but gets shifted back).
+                        all_tasks.insert(index, new_task)
+                        index += 1
+        bb.debug(3, 'merged task list: %s'  % all_tasks)
+
+        # Now reverse the order so that tasks that finish the work on one
+        # recipe are considered more imporant (= come first). The ordering
+        # is now so that do_build is most important.
+        all_tasks.reverse()
+
+        # Group tasks of the same kind before tasks of less important
+        # kinds at the head of the queue (because earlier = lower
+        # priority number = runs earlier), while preserving the
+        # ordering by recipe. If recipe foo is more important than
+        # bar, then the goal is to work on foo's do_populate_sysroot
+        # before bar's do_populate_sysroot and on the more important
+        # tasks of foo before any of the less important tasks in any
+        # other recipe (if those other recipes are more important than
+        # foo).
+        #
+        # All of this only applies when tasks are runable. Explicit
+        # dependencies still override this ordering by priority.
+        #
+        # Here's an example why this priority re-ordering helps with
+        # minimizing disk usage. Consider a recipe foo with a higher
+        # priority than bar where foo DEPENDS on bar. Then the
+        # implicit rule (from base.bbclass) is that foo's do_configure
+        # depends on bar's do_populate_sysroot. This ensures that
+        # bar's do_populate_sysroot gets done first. Normally the
+        # tasks from foo would continue to run once that is done, and
+        # bar only gets completed and cleaned up later. By ordering
+        # bar's task that depend on bar's do_populate_sysroot before foo's
+        # do_configure, that problem gets avoided.
+        task_index = 0
+        self.dump_prio('original priorities')
+        for task in all_tasks:
+            for index in range(task_index, self.numTasks):
+                taskid = self.prio_map[index]
+                taskname = taskid.rsplit(':', 1)[1]
+                if taskname == task:
+                    del self.prio_map[index]
+                    self.prio_map.insert(task_index, taskid)
+                    task_index += 1
+        self.dump_prio('completion priorities')
 
 class RunTaskEntry(object):
     def __init__(self):
@@ -262,10 +340,11 @@
         self.rq = rq
         self.warn_multi_bb = False
 
-        self.stampwhitelist = cfgData.getVar("BB_STAMP_WHITELIST", True) or ""
-        self.multi_provider_whitelist = (cfgData.getVar("MULTI_PROVIDER_WHITELIST", True) or "").split()
+        self.stampwhitelist = cfgData.getVar("BB_STAMP_WHITELIST") or ""
+        self.multi_provider_whitelist = (cfgData.getVar("MULTI_PROVIDER_WHITELIST") or "").split()
         self.setscenewhitelist = get_setscene_enforce_whitelist(cfgData)
         self.setscenewhitelist_checked = False
+        self.setscene_enforce = (cfgData.getVar('BB_SETSCENE_ENFORCE') == "1")
         self.init_progress_reporter = bb.progress.DummyMultiStageProcessProgressReporter()
 
         self.reset()
@@ -565,6 +644,8 @@
                 for (depname, idependtask) in irdepends:
                     if depname in taskData[mc].run_targets:
                         # Won't be in run_targets if ASSUME_PROVIDED
+                        if not taskData[mc].run_targets[depname]:
+                            continue
                         depdata = taskData[mc].run_targets[depname][0]
                         if depdata is not None:
                             t = depdata + ":" + idependtask
@@ -616,6 +697,9 @@
                 seendeps.add(t)
                 newdeps.add(t)
                 for i in newdeps:
+                    if i not in self.runtaskentries:
+                        # Not all recipes might have the recrdeptask task as a task
+                        continue
                     task = self.runtaskentries[i].task
                     for n in self.runtaskentries[i].depends:
                         if n not in seendeps:
@@ -722,6 +806,23 @@
 
         self.init_progress_reporter.next_stage()
 
+        if self.cooker.configuration.runall is not None:
+            runall = "do_%s" % self.cooker.configuration.runall
+            runall_tids = { k: v for k, v in self.runtaskentries.items() if taskname_from_tid(k) == runall }
+
+            # re-run the mark_active and then drop unused tasks from new list
+            runq_build = {}
+            for tid in list(runall_tids):
+                mark_active(tid,1)
+
+            for tid in list(self.runtaskentries.keys()):
+                if tid not in runq_build:
+                    del self.runtaskentries[tid]
+                    delcount += 1
+
+            if len(self.runtaskentries) == 0:
+                bb.msg.fatal("RunQueue", "No remaining tasks to run for build target %s with runall %s" % (target, runall))
+
         #
         # Step D - Sanity checks and computation
         #
@@ -976,16 +1077,22 @@
         self.cfgData = cfgData
         self.rqdata = RunQueueData(self, cooker, cfgData, dataCaches, taskData, targets)
 
-        self.stamppolicy = cfgData.getVar("BB_STAMP_POLICY", True) or "perfile"
-        self.hashvalidate = cfgData.getVar("BB_HASHCHECK_FUNCTION", True) or None
-        self.setsceneverify = cfgData.getVar("BB_SETSCENE_VERIFY_FUNCTION2", True) or None
-        self.depvalidate = cfgData.getVar("BB_SETSCENE_DEPVALID", True) or None
+        self.stamppolicy = cfgData.getVar("BB_STAMP_POLICY") or "perfile"
+        self.hashvalidate = cfgData.getVar("BB_HASHCHECK_FUNCTION") or None
+        self.setsceneverify = cfgData.getVar("BB_SETSCENE_VERIFY_FUNCTION2") or None
+        self.depvalidate = cfgData.getVar("BB_SETSCENE_DEPVALID") or None
 
         self.state = runQueuePrepare
 
         # For disk space monitor
+        # Invoked at regular time intervals via the bitbake heartbeat event
+        # while the build is running. We generate a unique name for the handler
+        # here, just in case that there ever is more than one RunQueue instance,
+        # start the handler when reaching runQueueSceneRun, and stop it when
+        # done with the build.
         self.dm = monitordisk.diskMonitor(cfgData)
-
+        self.dm_event_handler_name = '_bb_diskmonitor_' + str(id(self))
+        self.dm_event_handler_registered = False
         self.rqexe = None
         self.worker = {}
         self.fakeworker = {}
@@ -998,8 +1105,8 @@
         if fakeroot:
             magic = magic + "beef"
             mcdata = self.cooker.databuilder.mcdata[mc]
-            fakerootcmd = mcdata.getVar("FAKEROOTCMD", True)
-            fakerootenv = (mcdata.getVar("FAKEROOTBASEENV", True) or "").split()
+            fakerootcmd = mcdata.getVar("FAKEROOTCMD")
+            fakerootenv = (mcdata.getVar("FAKEROOTBASEENV") or "").split()
             env = os.environ.copy()
             for key, value in (var.split('=') for var in fakerootenv):
                 env[key] = value
@@ -1025,12 +1132,13 @@
             "logdefaultverboselogs" : bb.msg.loggerVerboseLogs,
             "logdefaultdomain" : bb.msg.loggerDefaultDomains,
             "prhost" : self.cooker.prhost,
-            "buildname" : self.cfgData.getVar("BUILDNAME", True),
-            "date" : self.cfgData.getVar("DATE", True),
-            "time" : self.cfgData.getVar("TIME", True),
+            "buildname" : self.cfgData.getVar("BUILDNAME"),
+            "date" : self.cfgData.getVar("DATE"),
+            "time" : self.cfgData.getVar("TIME"),
         }
 
         worker.stdin.write(b"<cookerconfig>" + pickle.dumps(self.cooker.configuration) + b"</cookerconfig>")
+        worker.stdin.write(b"<extraconfigdata>" + pickle.dumps(self.cooker.extraconfigdata) + b"</extraconfigdata>")
         worker.stdin.write(b"<workerdata>" + pickle.dumps(workerdata) + b"</workerdata>")
         worker.stdin.flush()
 
@@ -1208,10 +1316,12 @@
                 self.rqdata.init_progress_reporter.next_stage()
                 self.rqexe = RunQueueExecuteScenequeue(self)
 
-        if self.state in [runQueueSceneRun, runQueueRunning, runQueueCleanUp]:
-            self.dm.check(self)
-
         if self.state is runQueueSceneRun:
+            if not self.dm_event_handler_registered:
+                 res = bb.event.register(self.dm_event_handler_name,
+                                         lambda x: self.dm.check(self) if self.state in [runQueueSceneRun, runQueueRunning, runQueueCleanUp] else False,
+                                         ('bb.event.HeartbeatEvent',))
+                 self.dm_event_handler_registered = True
             retval = self.rqexe.execute()
 
         if self.state is runQueueRunInit:
@@ -1230,7 +1340,13 @@
         if self.state is runQueueCleanUp:
             retval = self.rqexe.finish()
 
-        if (self.state is runQueueComplete or self.state is runQueueFailed) and self.rqexe:
+        build_done = self.state is runQueueComplete or self.state is runQueueFailed
+
+        if build_done and self.dm_event_handler_registered:
+            bb.event.remove(self.dm_event_handler_name, None)
+            self.dm_event_handler_registered = False
+
+        if build_done and self.rqexe:
             self.teardown_workers()
             if self.rqexe.stats.failed:
                 logger.info("Tasks Summary: Attempted %d tasks of which %d didn't need to be rerun and %d failed.", self.rqexe.stats.completed + self.rqexe.stats.failed, self.rqexe.stats.skipped, self.rqexe.stats.failed)
@@ -1287,15 +1403,36 @@
         else:
             self.rqexe.finish()
 
-    def dump_signatures(self, options):
-        done = set()
-        bb.note("Reparsing files to collect dependency data")
+    def rq_dump_sigfn(self, fn, options):
         bb_cache = bb.cache.NoCache(self.cooker.databuilder)
+        the_data = bb_cache.loadDataFull(fn, self.cooker.collection.get_file_appends(fn))
+        siggen = bb.parse.siggen
+        dataCaches = self.rqdata.dataCaches
+        siggen.dump_sigfn(fn, dataCaches, options)
+
+    def dump_signatures(self, options):
+        fns = set()
+        bb.note("Reparsing files to collect dependency data")
+
         for tid in self.rqdata.runtaskentries:
             fn = fn_from_tid(tid)
-            if fn not in done:
-                the_data = bb_cache.loadDataFull(fn, self.cooker.collection.get_file_appends(fn))
-                done.add(fn)
+            fns.add(fn)
+
+        max_process = int(self.cfgData.getVar("BB_NUMBER_PARSE_THREADS") or os.cpu_count() or 1)
+        # We cannot use the real multiprocessing.Pool easily due to some local data
+        # that can't be pickled. This is a cheap multi-process solution.
+        launched = []
+        while fns:
+            if len(launched) < max_process:
+                p = Process(target=self.rq_dump_sigfn, args=(fns.pop(), options))
+                p.start()
+                launched.append(p)
+            for q in launched:
+                # The finished processes are joined when calling is_alive()
+                if not q.is_alive():
+                    launched.remove(q)
+        for p in launched:
+                p.join()
 
         bb.parse.siggen.dump_sigs(self.rqdata.dataCaches, options)
 
@@ -1326,7 +1463,7 @@
             sq_hash.append(self.rqdata.runtaskentries[tid].hash)
             sq_taskname.append(taskname)
             sq_task.append(tid)
-        locs = { "sq_fn" : sq_fn, "sq_task" : sq_taskname, "sq_hash" : sq_hash, "sq_hashfn" : sq_hashfn, "d" : self.cooker.expanded_data }
+        locs = { "sq_fn" : sq_fn, "sq_task" : sq_taskname, "sq_hash" : sq_hash, "sq_hashfn" : sq_hashfn, "d" : self.cooker.data }
         try:
             call = self.hashvalidate + "(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=True)"
             valid = bb.utils.better_eval(call, locs)
@@ -1427,8 +1564,8 @@
         self.cfgData = rq.cfgData
         self.rqdata = rq.rqdata
 
-        self.number_tasks = int(self.cfgData.getVar("BB_NUMBER_THREADS", True) or 1)
-        self.scheduler = self.cfgData.getVar("BB_SCHEDULER", True) or "speed"
+        self.number_tasks = int(self.cfgData.getVar("BB_NUMBER_THREADS") or 1)
+        self.scheduler = self.cfgData.getVar("BB_SCHEDULER") or "speed"
 
         self.runq_buildable = set()
         self.runq_running = set()
@@ -1510,7 +1647,7 @@
             pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn]
             taskdata[dep] = [pn, taskname, fn]
         call = self.rq.depvalidate + "(task, taskdata, notneeded, d)"
-        locs = { "task" : task, "taskdata" : taskdata, "notneeded" : self.scenequeue_notneeded, "d" : self.cooker.expanded_data }
+        locs = { "task" : task, "taskdata" : taskdata, "notneeded" : self.scenequeue_notneeded, "d" : self.cooker.data }
         valid = bb.utils.better_eval(call, locs)
         return valid
 
@@ -1578,7 +1715,7 @@
                 invalidtasks.append(tid)
 
             call = self.rq.setsceneverify + "(covered, tasknames, fns, d, invalidtasks=invalidtasks)"
-            locs = { "covered" : self.rq.scenequeue_covered, "tasknames" : tasknames, "fns" : fns, "d" : self.cooker.expanded_data, "invalidtasks" : invalidtasks }
+            locs = { "covered" : self.rq.scenequeue_covered, "tasknames" : tasknames, "fns" : fns, "d" : self.cooker.data, "invalidtasks" : invalidtasks }
             covered_remove = bb.utils.better_eval(call, locs)
 
         def removecoveredtask(tid):
@@ -1630,7 +1767,7 @@
                              if type(obj) is type and
                                 issubclass(obj, RunQueueScheduler))
 
-        user_schedulers = self.cfgData.getVar("BB_SCHEDULERS", True)
+        user_schedulers = self.cfgData.getVar("BB_SCHEDULERS")
         if user_schedulers:
             for sched in user_schedulers.split():
                 if not "." in sched:
@@ -1775,7 +1912,7 @@
                 bb.event.fire(startevent, self.cfgData)
                 self.runq_running.add(task)
                 self.stats.taskActive()
-                if not self.cooker.configuration.dry_run:
+                if not (self.cooker.configuration.dry_run or self.rqdata.setscene_enforce):
                     bb.build.make_stamp(taskname, self.rqdata.dataCaches[mc], taskfn)
                 self.task_complete(task)
                 return True
@@ -1786,7 +1923,7 @@
             taskdepdata = self.build_taskdepdata(task)
 
             taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn]
-            if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run:
+            if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not (self.cooker.configuration.dry_run or self.rqdata.setscene_enforce):
                 if not mc in self.rq.fakeworker:
                     try:
                         self.rq.start_fakeworker(self, mc)
@@ -1795,10 +1932,10 @@
                         self.rq.state = runQueueFailed
                         self.stats.taskFailed()
                         return True
-                self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, False, self.cooker.collection.get_file_appends(fn), taskdepdata)) + b"</runtask>")
+                self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, False, self.cooker.collection.get_file_appends(fn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>")
                 self.rq.fakeworker[mc].process.stdin.flush()
             else:
-                self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, False, self.cooker.collection.get_file_appends(taskfn), taskdepdata)) + b"</runtask>")
+                self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, False, self.cooker.collection.get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>")
                 self.rq.worker[mc].process.stdin.flush()
 
             self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True)
@@ -1839,7 +1976,8 @@
                 pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn]
                 deps = self.rqdata.runtaskentries[revdep].depends
                 provides = self.rqdata.dataCaches[mc].fn_provides[taskfn]
-                taskdepdata[revdep] = [pn, taskname, fn, deps, provides]
+                taskhash = self.rqdata.runtaskentries[revdep].hash
+                taskdepdata[revdep] = [pn, taskname, fn, deps, provides, taskhash]
                 for revdep2 in deps:
                     if revdep2 not in taskdepdata:
                         additional.append(revdep2)
@@ -1892,6 +2030,8 @@
         for tid in self.rqdata.runq_setscene_tids:
             #bb.warn("Added endpoint 2 %s" % (tid))
             for dep in self.rqdata.runtaskentries[tid].depends:
+                    if tid in sq_revdeps[dep]:
+                        sq_revdeps[dep].remove(tid)
                     if dep not in endpoints:
                         endpoints[dep] = set()
                     #bb.warn("  Added endpoint 3 %s" % (dep))
@@ -1911,12 +2051,13 @@
                 if point in self.rqdata.runq_setscene_tids:
                     sq_revdeps_new[point] = tasks
                     tasks = set()
+                    continue
                 for dep in self.rqdata.runtaskentries[point].depends:
                     if point in sq_revdeps[dep]:
                         sq_revdeps[dep].remove(point)
                     if tasks:
                         sq_revdeps_new[dep] |= tasks
-                    if (len(sq_revdeps[dep]) == 0 or len(sq_revdeps_new[dep]) != 0) and dep not in self.rqdata.runq_setscene_tids:
+                    if len(sq_revdeps[dep]) == 0 and dep not in self.rqdata.runq_setscene_tids:
                         newendpoints[dep] = task
             if len(newendpoints) != 0:
                 process_endpoints(newendpoints)
@@ -2072,7 +2213,7 @@
                 sq_taskname.append(taskname)
                 sq_task.append(tid)
             call = self.rq.hashvalidate + "(sq_fn, sq_task, sq_hash, sq_hashfn, d)"
-            locs = { "sq_fn" : sq_fn, "sq_task" : sq_taskname, "sq_hash" : sq_hash, "sq_hashfn" : sq_hashfn, "d" : self.cooker.expanded_data }
+            locs = { "sq_fn" : sq_fn, "sq_task" : sq_taskname, "sq_hash" : sq_hash, "sq_hashfn" : sq_hashfn, "d" : self.cooker.data }
             valid = bb.utils.better_eval(call, locs)
 
             valid_new = stamppresent
@@ -2199,14 +2340,16 @@
             startevent = sceneQueueTaskStarted(task, self.stats, self.rq)
             bb.event.fire(startevent, self.cfgData)
 
+            taskdepdata = self.build_taskdepdata(task)
+
             taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn]
             if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run:
                 if not mc in self.rq.fakeworker:
                     self.rq.start_fakeworker(self, mc)
-                self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, True, self.cooker.collection.get_file_appends(taskfn), None)) + b"</runtask>")
+                self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>")
                 self.rq.fakeworker[mc].process.stdin.flush()
             else:
-                self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, True, self.cooker.collection.get_file_appends(taskfn), None)) + b"</runtask>")
+                self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>")
                 self.rq.worker[mc].process.stdin.flush()
 
             self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True)
@@ -2241,6 +2384,44 @@
     def runqueue_process_waitpid(self, task, status):
         RunQueueExecute.runqueue_process_waitpid(self, task, status)
 
+
+    def build_taskdepdata(self, task):
+        def getsetscenedeps(tid):
+            deps = set()
+            (mc, fn, taskname, _) = split_tid_mcfn(tid)
+            realtid = tid + "_setscene"
+            idepends = self.rqdata.taskData[mc].taskentries[realtid].idepends
+            for (depname, idependtask) in idepends:
+                if depname not in self.rqdata.taskData[mc].build_targets:
+                    continue
+
+                depfn = self.rqdata.taskData[mc].build_targets[depname][0]
+                if depfn is None:
+                     continue
+                deptid = depfn + ":" + idependtask.replace("_setscene", "")
+                deps.add(deptid)
+            return deps
+
+        taskdepdata = {}
+        next = getsetscenedeps(task)
+        next.add(task)
+        while next:
+            additional = []
+            for revdep in next:
+                (mc, fn, taskname, taskfn) = split_tid_mcfn(revdep)
+                pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn]
+                deps = getsetscenedeps(revdep)
+                provides = self.rqdata.dataCaches[mc].fn_provides[taskfn]
+                taskhash = self.rqdata.runtaskentries[revdep].hash
+                taskdepdata[revdep] = [pn, taskname, fn, deps, provides, taskhash]
+                for revdep2 in deps:
+                    if revdep2 not in taskdepdata:
+                        additional.append(revdep2)
+            next = additional
+
+        #bb.note("Task %s: " % task + str(taskdepdata).replace("], ", "],\n"))
+        return taskdepdata
+
 class TaskFailure(Exception):
     """
     Exception raised when a task in a runqueue fails
@@ -2406,9 +2587,9 @@
         self.input.close()
 
 def get_setscene_enforce_whitelist(d):
-    if d.getVar('BB_SETSCENE_ENFORCE', True) != '1':
+    if d.getVar('BB_SETSCENE_ENFORCE') != '1':
         return None
-    whitelist = (d.getVar("BB_SETSCENE_ENFORCE_WHITELIST", True) or "").split()
+    whitelist = (d.getVar("BB_SETSCENE_ENFORCE_WHITELIST") or "").split()
     outlist = []
     for item in whitelist[:]:
         if item.startswith('%:'):
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/server/process.py b/import-layers/yocto-poky/bitbake/lib/bb/server/process.py
index 982fcf7..c3c1450 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/server/process.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/server/process.py
@@ -92,6 +92,8 @@
         self.event = EventAdapter(event_queue)
         self.featurelist = featurelist
         self.quit = False
+        self.heartbeat_seconds = 1 # default, BB_HEARTBEAT_EVENT will be checked once we have a datastore.
+        self.next_heartbeat = time.time()
 
         self.quitin, self.quitout = Pipe()
         self.event_handle = multiprocessing.Value("i")
@@ -101,6 +103,14 @@
             self.event_queue.put(event)
         self.event_handle.value = bb.event.register_UIHhandler(self, True)
 
+        heartbeat_event = self.cooker.data.getVar('BB_HEARTBEAT_EVENT')
+        if heartbeat_event:
+            try:
+                self.heartbeat_seconds = float(heartbeat_event)
+            except:
+                # Throwing an exception here causes bitbake to hang.
+                # Just warn about the invalid setting and continue
+                bb.warn('Ignoring invalid BB_HEARTBEAT_EVENT=%s, must be a float specifying seconds.' % heartbeat_event)
         bb.cooker.server_main(self.cooker, self.main)
 
     def main(self):
@@ -160,6 +170,21 @@
                 del self._idlefuns[function]
                 self.quit = True
 
+        # Create new heartbeat event?
+        now = time.time()
+        if now >= self.next_heartbeat:
+            # We might have missed heartbeats. Just trigger once in
+            # that case and continue after the usual delay.
+            self.next_heartbeat += self.heartbeat_seconds
+            if self.next_heartbeat <= now:
+                self.next_heartbeat = now + self.heartbeat_seconds
+            heartbeat = bb.event.HeartbeatEvent(now)
+            bb.event.fire(heartbeat, self.cooker.data)
+        if nextsleep and now + nextsleep > self.next_heartbeat:
+            # Shorten timeout so that we we wake up in time for
+            # the heartbeat.
+            nextsleep = self.next_heartbeat - now
+
         if nextsleep is not None:
             select.select(fds,[],[],nextsleep)
 
@@ -199,7 +224,6 @@
                 if isinstance(event, logging.LogRecord):
                     logger.handle(event)
 
-        signal.signal(signal.SIGINT, signal.SIG_IGN)
         self.procserver.stop()
 
         while self.procserver.is_alive():
@@ -209,6 +233,9 @@
         self.ui_channel.close()
         self.event_queue.close()
         self.event_queue.setexit()
+        # XXX: Call explicity close in _writer to avoid
+        # fd leakage because isn't called on Queue.close()
+        self.event_queue._writer.close()
 
 # Wrap Queue to provide API which isn't server implementation specific
 class ProcessEventQueue(multiprocessing.queues.Queue):
@@ -240,7 +267,6 @@
                 sys.exit(1)
             return None
 
-
 class BitBakeServer(BitBakeBaseServer):
     def initServer(self, single_use=True):
         # establish communication channels.  We use bidirectional pipes for
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/server/xmlrpc.py b/import-layers/yocto-poky/bitbake/lib/bb/server/xmlrpc.py
index 452f14b..a06007f 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/server/xmlrpc.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/server/xmlrpc.py
@@ -190,7 +190,7 @@
         self.send_header("Content-type", "text/plain")
         self.send_header("Content-length", str(len(response)))
         self.end_headers()
-        self.wfile.write(response)
+        self.wfile.write(bytes(response, 'utf-8'))
 
 
 class XMLRPCProxyServer(BaseImplServer):
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/siggen.py b/import-layers/yocto-poky/bitbake/lib/bb/siggen.py
index 542bbb9..f71190a 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/siggen.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/siggen.py
@@ -5,6 +5,8 @@
 import tempfile
 import pickle
 import bb.data
+import difflib
+import simplediff
 from bb.checksum import FileChecksumCache
 
 logger = logging.getLogger('BitBake.SigGen')
@@ -13,7 +15,7 @@
     siggens = [obj for obj in globals().values()
                       if type(obj) is type and issubclass(obj, SignatureGenerator)]
 
-    desired = d.getVar("BB_SIGNATURE_HANDLER", True) or "noop"
+    desired = d.getVar("BB_SIGNATURE_HANDLER") or "noop"
     for sg in siggens:
         if desired == sg.name:
             return sg(d)
@@ -82,10 +84,10 @@
         self.gendeps = {}
         self.lookupcache = {}
         self.pkgnameextract = re.compile("(?P<fn>.*)\..*")
-        self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST", True) or "").split())
+        self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST") or "").split())
         self.taskwhitelist = None
         self.init_rundepcheck(data)
-        checksum_cache_file = data.getVar("BB_HASH_CHECKSUM_CACHE_FILE", True)
+        checksum_cache_file = data.getVar("BB_HASH_CHECKSUM_CACHE_FILE")
         if checksum_cache_file:
             self.checksum_cache = FileChecksumCache()
             self.checksum_cache.init_cache(data, checksum_cache_file)
@@ -93,7 +95,7 @@
             self.checksum_cache = None
 
     def init_rundepcheck(self, data):
-        self.taskwhitelist = data.getVar("BB_HASHTASK_WHITELIST", True) or None
+        self.taskwhitelist = data.getVar("BB_HASHTASK_WHITELIST") or None
         if self.taskwhitelist:
             self.twl = re.compile(self.taskwhitelist)
         else:
@@ -101,6 +103,7 @@
 
     def _build_data(self, fn, d):
 
+        ignore_mismatch = ((d.getVar("BB_HASH_IGNORE_MISMATCH") or '') == '1')
         tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d)
 
         taskdeps = {}
@@ -135,7 +138,7 @@
                     data = data + str(var)
             datahash = hashlib.md5(data.encode("utf-8")).hexdigest()
             k = fn + "." + task
-            if k in self.basehash and self.basehash[k] != datahash:
+            if not ignore_mismatch and k in self.basehash and self.basehash[k] != datahash:
                 bb.error("When reparsing %s, the basehash value changed from %s to %s. The metadata is not deterministic and this needs to be fixed." % (k, self.basehash[k], datahash))
             self.basehash[k] = datahash
             taskdeps[task] = alldeps
@@ -154,13 +157,15 @@
 
         try:
             taskdeps = self._build_data(fn, d)
+        except bb.parse.SkipRecipe:
+            raise
         except:
             bb.warn("Error during finalise of %s" % fn)
             raise
 
         #Slow but can be useful for debugging mismatched basehashes
         #for task in self.taskdeps[fn]:
-        #    self.dump_sigtask(fn, task, d.getVar("STAMP", True), False)
+        #    self.dump_sigtask(fn, task, d.getVar("STAMP"), False)
 
         for task in taskdeps:
             d.setVar("BB_BASEHASH_task-%s" % task, self.basehash[fn + "." + task])
@@ -306,8 +311,8 @@
                 pass
             raise err
 
-    def dump_sigs(self, dataCaches, options):
-        for fn in self.taskdeps:
+    def dump_sigfn(self, fn, dataCaches, options):
+        if fn in self.taskdeps:
             for task in self.taskdeps[fn]:
                 tid = fn + ":" + task
                 (mc, _, _) = bb.runqueue.split_tid(tid)
@@ -345,16 +350,67 @@
 
 def dump_this_task(outfile, d):
     import bb.parse
-    fn = d.getVar("BB_FILENAME", True)
-    task = "do_" + d.getVar("BB_CURRENTTASK", True)
+    fn = d.getVar("BB_FILENAME")
+    task = "do_" + d.getVar("BB_CURRENTTASK")
     referencestamp = bb.build.stamp_internal(task, d, None, True)
     bb.parse.siggen.dump_sigtask(fn, task, outfile, "customfile:" + referencestamp)
 
+def init_colors(enable_color):
+    """Initialise colour dict for passing to compare_sigfiles()"""
+    # First set up the colours
+    colors = {'color_title':   '\033[1;37;40m',
+              'color_default': '\033[0;37;40m',
+              'color_add':     '\033[1;32;40m',
+              'color_remove':  '\033[1;31;40m',
+             }
+    # Leave all keys present but clear the values
+    if not enable_color:
+        for k in colors.keys():
+            colors[k] = ''
+    return colors
+
+def worddiff_str(oldstr, newstr, colors=None):
+    if not colors:
+        colors = init_colors(False)
+    diff = simplediff.diff(oldstr.split(' '), newstr.split(' '))
+    ret = []
+    for change, value in diff:
+        value = ' '.join(value)
+        if change == '=':
+            ret.append(value)
+        elif change == '+':
+            item = '{color_add}{{+{value}+}}{color_default}'.format(value=value, **colors)
+            ret.append(item)
+        elif change == '-':
+            item = '{color_remove}[-{value}-]{color_default}'.format(value=value, **colors)
+            ret.append(item)
+    whitespace_note = ''
+    if oldstr != newstr and ' '.join(oldstr.split()) == ' '.join(newstr.split()):
+        whitespace_note = ' (whitespace changed)'
+    return '"%s"%s' % (' '.join(ret), whitespace_note)
+
+def list_inline_diff(oldlist, newlist, colors=None):
+    if not colors:
+        colors = init_colors(False)
+    diff = simplediff.diff(oldlist, newlist)
+    ret = []
+    for change, value in diff:
+        value = ' '.join(value)
+        if change == '=':
+            ret.append("'%s'" % value)
+        elif change == '+':
+            item = '{color_add}+{value}{color_default}'.format(value=value, **colors)
+            ret.append(item)
+        elif change == '-':
+            item = '{color_remove}-{value}{color_default}'.format(value=value, **colors)
+            ret.append(item)
+    return '[%s]' % (', '.join(ret))
+
 def clean_basepath(a):
     mc = None
     if a.startswith("multiconfig:"):
         _, mc, a = a.split(":", 2)
-    b = a.rsplit("/", 2)[1] + a.rsplit("/", 2)[2]
+    b = a.rsplit("/", 2)[1] + '/' + a.rsplit("/", 2)[2]
     if a.startswith("virtual:"):
         b = b + ":" + a.rsplit(":", 1)[0]
     if mc:
@@ -373,9 +429,26 @@
         b.append(clean_basepath(x))
     return b
 
-def compare_sigfiles(a, b, recursecb = None):
+def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
     output = []
 
+    colors = init_colors(color)
+    def color_format(formatstr, **values):
+        """
+        Return colour formatted string.
+        NOTE: call with the format string, not an already formatted string
+        containing values (otherwise you could have trouble with { and }
+        characters)
+        """
+        if not formatstr.endswith('{color_default}'):
+            formatstr += '{color_default}'
+        # In newer python 3 versions you can pass both of these directly,
+        # but we only require 3.4 at the moment
+        formatparams = {}
+        formatparams.update(colors)
+        formatparams.update(values)
+        return formatstr.format(**formatparams)
+
     with open(a, 'rb') as f:
         p1 = pickle.Unpickler(f)
         a_data = p1.load()
@@ -429,39 +502,59 @@
         return changed, added, removed
 
     if 'basewhitelist' in a_data and a_data['basewhitelist'] != b_data['basewhitelist']:
-        output.append("basewhitelist changed from '%s' to '%s'" % (a_data['basewhitelist'], b_data['basewhitelist']))
+        output.append(color_format("{color_title}basewhitelist changed{color_default} from '%s' to '%s'") % (a_data['basewhitelist'], b_data['basewhitelist']))
         if a_data['basewhitelist'] and b_data['basewhitelist']:
             output.append("changed items: %s" % a_data['basewhitelist'].symmetric_difference(b_data['basewhitelist']))
 
     if 'taskwhitelist' in a_data and a_data['taskwhitelist'] != b_data['taskwhitelist']:
-        output.append("taskwhitelist changed from '%s' to '%s'" % (a_data['taskwhitelist'], b_data['taskwhitelist']))
+        output.append(color_format("{color_title}taskwhitelist changed{color_default} from '%s' to '%s'") % (a_data['taskwhitelist'], b_data['taskwhitelist']))
         if a_data['taskwhitelist'] and b_data['taskwhitelist']:
             output.append("changed items: %s" % a_data['taskwhitelist'].symmetric_difference(b_data['taskwhitelist']))
 
     if a_data['taskdeps'] != b_data['taskdeps']:
-        output.append("Task dependencies changed from:\n%s\nto:\n%s" % (sorted(a_data['taskdeps']), sorted(b_data['taskdeps'])))
+        output.append(color_format("{color_title}Task dependencies changed{color_default} from:\n%s\nto:\n%s") % (sorted(a_data['taskdeps']), sorted(b_data['taskdeps'])))
 
-    if a_data['basehash'] != b_data['basehash']:
-        output.append("basehash changed from %s to %s" % (a_data['basehash'], b_data['basehash']))
+    if a_data['basehash'] != b_data['basehash'] and not collapsed:
+        output.append(color_format("{color_title}basehash changed{color_default} from %s to %s") % (a_data['basehash'], b_data['basehash']))
 
     changed, added, removed = dict_diff(a_data['gendeps'], b_data['gendeps'], a_data['basewhitelist'] & b_data['basewhitelist'])
     if changed:
         for dep in changed:
-            output.append("List of dependencies for variable %s changed from '%s' to '%s'" % (dep, a_data['gendeps'][dep], b_data['gendeps'][dep]))
+            output.append(color_format("{color_title}List of dependencies for variable %s changed from '{color_default}%s{color_title}' to '{color_default}%s{color_title}'") % (dep, a_data['gendeps'][dep], b_data['gendeps'][dep]))
             if a_data['gendeps'][dep] and b_data['gendeps'][dep]:
                 output.append("changed items: %s" % a_data['gendeps'][dep].symmetric_difference(b_data['gendeps'][dep]))
     if added:
         for dep in added:
-            output.append("Dependency on variable %s was added" % (dep))
+            output.append(color_format("{color_title}Dependency on variable %s was added") % (dep))
     if removed:
         for dep in removed:
-            output.append("Dependency on Variable %s was removed" % (dep))
+            output.append(color_format("{color_title}Dependency on Variable %s was removed") % (dep))
 
 
     changed, added, removed = dict_diff(a_data['varvals'], b_data['varvals'])
     if changed:
         for dep in changed:
-            output.append("Variable %s value changed from '%s' to '%s'" % (dep, a_data['varvals'][dep], b_data['varvals'][dep]))
+            oldval = a_data['varvals'][dep]
+            newval = b_data['varvals'][dep]
+            if newval and oldval and ('\n' in oldval or '\n' in newval):
+                diff = difflib.unified_diff(oldval.splitlines(), newval.splitlines(), lineterm='')
+                # Cut off the first two lines, since we aren't interested in
+                # the old/new filename (they are blank anyway in this case)
+                difflines = list(diff)[2:]
+                if color:
+                    # Add colour to diff output
+                    for i, line in enumerate(difflines):
+                        if line.startswith('+'):
+                            line = color_format('{color_add}{line}', line=line)
+                            difflines[i] = line
+                        elif line.startswith('-'):
+                            line = color_format('{color_remove}{line}', line=line)
+                            difflines[i] = line
+                output.append(color_format("{color_title}Variable {var} value changed:{color_default}\n{diff}", var=dep, diff='\n'.join(difflines)))
+            elif newval and oldval and (' ' in oldval or ' ' in newval):
+                output.append(color_format("{color_title}Variable {var} value changed:{color_default}\n{diff}", var=dep, diff=worddiff_str(oldval, newval, colors)))
+            else:
+                output.append(color_format("{color_title}Variable {var} value changed from '{color_default}{oldval}{color_title}' to '{color_default}{newval}{color_title}'{color_default}", var=dep, oldval=oldval, newval=newval))
 
     if not 'file_checksum_values' in a_data:
          a_data['file_checksum_values'] = {}
@@ -471,32 +564,38 @@
     changed, added, removed = file_checksums_diff(a_data['file_checksum_values'], b_data['file_checksum_values'])
     if changed:
         for f, old, new in changed:
-            output.append("Checksum for file %s changed from %s to %s" % (f, old, new))
+            output.append(color_format("{color_title}Checksum for file %s changed{color_default} from %s to %s") % (f, old, new))
     if added:
         for f in added:
-            output.append("Dependency on checksum of file %s was added" % (f))
+            output.append(color_format("{color_title}Dependency on checksum of file %s was added") % (f))
     if removed:
         for f in removed:
-            output.append("Dependency on checksum of file %s was removed" % (f))
+            output.append(color_format("{color_title}Dependency on checksum of file %s was removed") % (f))
 
     if not 'runtaskdeps' in a_data:
          a_data['runtaskdeps'] = {}
     if not 'runtaskdeps' in b_data:
          b_data['runtaskdeps'] = {}
 
-    if len(a_data['runtaskdeps']) != len(b_data['runtaskdeps']):
-        changed = ["Number of task dependencies changed"]
-    else:
-        changed = []
-        for idx, task in enumerate(a_data['runtaskdeps']):
-            a = a_data['runtaskdeps'][idx]
-            b = b_data['runtaskdeps'][idx]
-            if a_data['runtaskhashes'][a] != b_data['runtaskhashes'][b]:
-                changed.append("%s with hash %s\n changed to\n%s with hash %s" % (a, a_data['runtaskhashes'][a], b, b_data['runtaskhashes'][b]))
+    if not collapsed:
+        if len(a_data['runtaskdeps']) != len(b_data['runtaskdeps']):
+            changed = ["Number of task dependencies changed"]
+        else:
+            changed = []
+            for idx, task in enumerate(a_data['runtaskdeps']):
+                a = a_data['runtaskdeps'][idx]
+                b = b_data['runtaskdeps'][idx]
+                if a_data['runtaskhashes'][a] != b_data['runtaskhashes'][b] and not collapsed:
+                    changed.append("%s with hash %s\n changed to\n%s with hash %s" % (clean_basepath(a), a_data['runtaskhashes'][a], clean_basepath(b), b_data['runtaskhashes'][b]))
 
-    if changed:
-        output.append("runtaskdeps changed from %s to %s" % (clean_basepaths_list(a_data['runtaskdeps']), clean_basepaths_list(b_data['runtaskdeps'])))
-        output.append("\n".join(changed))
+        if changed:
+            clean_a = clean_basepaths_list(a_data['runtaskdeps'])
+            clean_b = clean_basepaths_list(b_data['runtaskdeps'])
+            if clean_a != clean_b:
+                output.append(color_format("{color_title}runtaskdeps changed:{color_default}\n%s") % list_inline_diff(clean_a, clean_b, colors))
+            else:
+                output.append(color_format("{color_title}runtaskdeps changed:"))
+            output.append("\n".join(changed))
 
 
     if 'runtaskhashes' in a_data and 'runtaskhashes' in b_data:
@@ -512,7 +611,7 @@
                             #output.append("Dependency on task %s was replaced by %s with same hash" % (dep, bdep))
                             bdep_found = True
                 if not bdep_found:
-                    output.append("Dependency on task %s was added with hash %s" % (clean_basepath(dep), b[dep]))
+                    output.append(color_format("{color_title}Dependency on task %s was added{color_default} with hash %s") % (clean_basepath(dep), b[dep]))
         if removed:
             for dep in removed:
                 adep_found = False
@@ -522,21 +621,25 @@
                             #output.append("Dependency on task %s was replaced by %s with same hash" % (adep, dep))
                             adep_found = True
                 if not adep_found:
-                    output.append("Dependency on task %s was removed with hash %s" % (clean_basepath(dep), a[dep]))
+                    output.append(color_format("{color_title}Dependency on task %s was removed{color_default} with hash %s") % (clean_basepath(dep), a[dep]))
         if changed:
             for dep in changed:
-                output.append("Hash for dependent task %s changed from %s to %s" % (clean_basepath(dep), a[dep], b[dep]))
+                if not collapsed:
+                    output.append(color_format("{color_title}Hash for dependent task %s changed{color_default} from %s to %s") % (clean_basepath(dep), a[dep], b[dep]))
                 if callable(recursecb):
-                    # If a dependent hash changed, might as well print the line above and then defer to the changes in 
-                    # that hash since in all likelyhood, they're the same changes this task also saw.
                     recout = recursecb(dep, a[dep], b[dep])
                     if recout:
-                        output = [output[-1]] + recout
+                        if collapsed:
+                            output.extend(recout)
+                        else:
+                            # If a dependent hash changed, might as well print the line above and then defer to the changes in 
+                            # that hash since in all likelyhood, they're the same changes this task also saw.
+                            output = [output[-1]] + recout
 
     a_taint = a_data.get('taint', None)
     b_taint = b_data.get('taint', None)
     if a_taint != b_taint:
-        output.append("Taint (by forced/invalidated task) changed from %s to %s" % (a_taint, b_taint))
+        output.append(color_format("{color_title}Taint (by forced/invalidated task) changed{color_default} from %s to %s") % (a_taint, b_taint))
 
     return output
 
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/taskdata.py b/import-layers/yocto-poky/bitbake/lib/bb/taskdata.py
index d8bdbca..8c96a56 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/taskdata.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/taskdata.py
@@ -89,6 +89,19 @@
 
         self.add_extra_deps(fn, dataCache)
 
+        # Common code for dep_name/depends = 'depends'/idepends and 'rdepends'/irdepends
+        def handle_deps(task, dep_name, depends, seen):
+            if dep_name in task_deps and task in task_deps[dep_name]:
+                ids = []
+                for dep in task_deps[dep_name][task].split():
+                    if dep:
+                        parts = dep.split(":")
+                        if len(parts) != 2:
+                            bb.msg.fatal("TaskData", "Error for %s:%s[%s], dependency %s in '%s' does not contain exactly one ':' character.\n Task '%s' should be specified in the form 'packagename:task'" % (fn, task, dep_name, dep, task_deps[dep_name][task], dep_name))
+                        ids.append((parts[0], parts[1]))
+                        seen(parts[0])
+                depends.extend(ids)
+
         for task in task_deps['tasks']:
 
             tid = "%s:%s" % (fn, task)
@@ -105,24 +118,8 @@
             self.taskentries[tid].tdepends.extend(parentids)
 
             # Touch all intertask dependencies
-            if 'depends' in task_deps and task in task_deps['depends']:
-                ids = []
-                for dep in task_deps['depends'][task].split():
-                    if dep:
-                        if ":" not in dep:
-                            bb.msg.fatal("TaskData", "Error for %s, dependency %s does not contain ':' character\n. Task 'depends' should be specified in the form 'packagename:task'" % (fn, dep))
-                        ids.append(((dep.split(":")[0]), dep.split(":")[1]))
-                        self.seen_build_target(dep.split(":")[0])
-                self.taskentries[tid].idepends.extend(ids)
-            if 'rdepends' in task_deps and task in task_deps['rdepends']:
-                ids = []
-                for dep in task_deps['rdepends'][task].split():
-                    if dep:
-                        if ":" not in dep:
-                            bb.msg.fatal("TaskData", "Error for %s, dependency %s does not contain ':' character\n. Task 'rdepends' should be specified in the form 'packagename:task'" % (fn, dep))
-                        ids.append(((dep.split(":")[0]), dep.split(":")[1]))
-                        self.seen_run_target(dep.split(":")[0])
-                self.taskentries[tid].irdepends.extend(ids)
+            handle_deps(task, 'depends', self.taskentries[tid].idepends, self.seen_build_target)
+            handle_deps(task, 'rdepends', self.taskentries[tid].irdepends, self.seen_run_target)
 
         # Work out build dependencies
         if not fn in self.depids:
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/tests/codeparser.py b/import-layers/yocto-poky/bitbake/lib/bb/tests/codeparser.py
index 14f0e25..e30e78c 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/tests/codeparser.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/tests/codeparser.py
@@ -49,6 +49,9 @@
     def assertExecs(self, execs):
         self.assertEqual(self.execs, execs)
 
+    def assertContains(self, contains):
+        self.assertEqual(self.contains, contains)
+
 class VariableReferenceTest(ReferenceTest):
 
     def parseExpression(self, exp):
@@ -68,7 +71,7 @@
 
     def test_python_reference(self):
         self.setEmptyVars(["BAR"])
-        self.parseExpression("${@bb.data.getVar('BAR', d, True) + 'foo'}")
+        self.parseExpression("${@d.getVar('BAR') + 'foo'}")
         self.assertReferences(set(["BAR"]))
 
 class ShellReferenceTest(ReferenceTest):
@@ -201,6 +204,7 @@
 
         self.references = parsedvar.references | parser.references
         self.execs = parser.execs
+        self.contains = parser.contains
 
     @staticmethod
     def indent(value):
@@ -209,17 +213,17 @@
         return " " + value
 
     def test_getvar_reference(self):
-        self.parseExpression("bb.data.getVar('foo', d, True)")
+        self.parseExpression("d.getVar('foo')")
         self.assertReferences(set(["foo"]))
         self.assertExecs(set())
 
     def test_getvar_computed_reference(self):
-        self.parseExpression("bb.data.getVar('f' + 'o' + 'o', d, True)")
+        self.parseExpression("d.getVar('f' + 'o' + 'o')")
         self.assertReferences(set())
         self.assertExecs(set())
 
     def test_getvar_exec_reference(self):
-        self.parseExpression("eval('bb.data.getVar(\"foo\", d, True)')")
+        self.parseExpression("eval('d.getVar(\"foo\")')")
         self.assertReferences(set())
         self.assertExecs(set(["eval"]))
 
@@ -265,15 +269,35 @@
         self.assertExecs(set(["testget"]))
         del self.context["testget"]
 
+    def test_contains(self):
+        self.parseExpression('bb.utils.contains("TESTVAR", "one", "true", "false", d)')
+        self.assertContains({'TESTVAR': {'one'}})
+
+    def test_contains_multi(self):
+        self.parseExpression('bb.utils.contains("TESTVAR", "one two", "true", "false", d)')
+        self.assertContains({'TESTVAR': {'one two'}})
+
+    def test_contains_any(self):
+        self.parseExpression('bb.utils.contains_any("TESTVAR", "hello", "true", "false", d)')
+        self.assertContains({'TESTVAR': {'hello'}})
+
+    def test_contains_any_multi(self):
+        self.parseExpression('bb.utils.contains_any("TESTVAR", "one two three", "true", "false", d)')
+        self.assertContains({'TESTVAR': {'one', 'two', 'three'}})
+
+    def test_contains_filter(self):
+        self.parseExpression('bb.utils.filter("TESTVAR", "hello there world", d)')
+        self.assertContains({'TESTVAR': {'hello', 'there', 'world'}})
+
 
 class DependencyReferenceTest(ReferenceTest):
 
     pydata = """
-bb.data.getVar('somevar', d, True)
+d.getVar('somevar')
 def test(d):
     foo = 'bar %s' % 'foo'
 def test2(d):
-    d.getVar(foo, True)
+    d.getVar(foo)
     d.getVar('bar', False)
     test2(d)
 
@@ -285,9 +309,9 @@
 
 test(d)
 
-bb.data.expand(bb.data.getVar("something", False, d), d)
-bb.data.expand("${inexpand} somethingelse", d)
-bb.data.getVar(a(), d, False)
+d.expand(d.getVar("something", False))
+d.expand("${inexpand} somethingelse")
+d.getVar(a(), False)
 """
 
     def test_python(self):
@@ -370,6 +394,30 @@
 
         self.assertEqual(deps, set(["oe_libinstall"]))
 
+    def test_contains_vardeps(self):
+        expr = '${@bb.utils.filter("TESTVAR", "somevalue anothervalue", d)} \
+                ${@bb.utils.contains("TESTVAR", "testval testval2", "yetanothervalue", "", d)} \
+                ${@bb.utils.contains("TESTVAR", "testval2 testval3", "blah", "", d)} \
+                ${@bb.utils.contains_any("TESTVAR", "testval2 testval3", "lastone", "", d)}'
+        parsedvar = self.d.expandWithRefs(expr, None)
+        # Check contains
+        self.assertEqual(parsedvar.contains, {'TESTVAR': {'testval2 testval3', 'anothervalue', 'somevalue', 'testval testval2', 'testval2', 'testval3'}})
+        # Check dependencies
+        self.d.setVar('ANOTHERVAR', expr)
+        self.d.setVar('TESTVAR', 'anothervalue testval testval2')
+        deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), self.d)
+        self.assertEqual(sorted(values.splitlines()),
+                         sorted([expr,
+                          'TESTVAR{anothervalue} = Set',
+                          'TESTVAR{somevalue} = Unset',
+                          'TESTVAR{testval testval2} = Set',
+                          'TESTVAR{testval2 testval3} = Unset',
+                          'TESTVAR{testval2} = Set',
+                          'TESTVAR{testval3} = Unset'
+                          ]))
+        # Check final value
+        self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['anothervalue', 'yetanothervalue', 'lastone'])
+
     #Currently no wildcard support
     #def test_vardeps_wildcards(self):
     #    self.d.setVar("oe_libinstall", "echo test")
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/tests/data.py b/import-layers/yocto-poky/bitbake/lib/bb/tests/data.py
index b54eb06..a4a9dd3 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/tests/data.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/tests/data.py
@@ -77,13 +77,13 @@
         self.assertEqual(str(val), "boo value_of_foo")
 
     def test_python_snippet_getvar(self):
-        val = self.d.expand("${@d.getVar('foo', True) + ' ${bar}'}")
+        val = self.d.expand("${@d.getVar('foo') + ' ${bar}'}")
         self.assertEqual(str(val), "value_of_foo value_of_bar")
 
     def test_python_unexpanded(self):
         self.d.setVar("bar", "${unsetvar}")
-        val = self.d.expand("${@d.getVar('foo', True) + ' ${bar}'}")
-        self.assertEqual(str(val), "${@d.getVar('foo', True) + ' ${unsetvar}'}")
+        val = self.d.expand("${@d.getVar('foo') + ' ${bar}'}")
+        self.assertEqual(str(val), "${@d.getVar('foo') + ' ${unsetvar}'}")
 
     def test_python_snippet_syntax_error(self):
         self.d.setVar("FOO", "${@foo = 5}")
@@ -99,7 +99,7 @@
         self.assertRaises(bb.data_smart.ExpansionError, self.d.getVar, "FOO", True)
 
     def test_value_containing_value(self):
-        val = self.d.expand("${@d.getVar('foo', True) + ' ${bar}'}")
+        val = self.d.expand("${@d.getVar('foo') + ' ${bar}'}")
         self.assertEqual(str(val), "value_of_foo value_of_bar")
 
     def test_reference_undefined_var(self):
@@ -109,7 +109,7 @@
     def test_double_reference(self):
         self.d.setVar("BAR", "bar value")
         self.d.setVar("FOO", "${BAR} foo ${BAR}")
-        val = self.d.getVar("FOO", True)
+        val = self.d.getVar("FOO")
         self.assertEqual(str(val), "bar value foo bar value")
 
     def test_direct_recursion(self):
@@ -129,12 +129,12 @@
 
     def test_incomplete_varexp_single_quotes(self):
         self.d.setVar("FOO", "sed -i -e 's:IP{:I${:g' $pc")
-        val = self.d.getVar("FOO", True)
+        val = self.d.getVar("FOO")
         self.assertEqual(str(val), "sed -i -e 's:IP{:I${:g' $pc")
 
     def test_nonstring(self):
         self.d.setVar("TEST", 5)
-        val = self.d.getVar("TEST", True)
+        val = self.d.getVar("TEST")
         self.assertEqual(str(val), "5")
 
     def test_rename(self):
@@ -234,19 +234,19 @@
     def test_prepend(self):
         self.d.setVar("TEST", "${VAL}")
         self.d.prependVar("TEST", "${FOO}:")
-        self.assertEqual(self.d.getVar("TEST", True), "foo:val")
+        self.assertEqual(self.d.getVar("TEST"), "foo:val")
 
     def test_append(self):
         self.d.setVar("TEST", "${VAL}")
         self.d.appendVar("TEST", ":${BAR}")
-        self.assertEqual(self.d.getVar("TEST", True), "val:bar")
+        self.assertEqual(self.d.getVar("TEST"), "val:bar")
 
     def test_multiple_append(self):
         self.d.setVar("TEST", "${VAL}")
         self.d.prependVar("TEST", "${FOO}:")
         self.d.appendVar("TEST", ":val2")
         self.d.appendVar("TEST", ":${BAR}")
-        self.assertEqual(self.d.getVar("TEST", True), "foo:val:val2:bar")
+        self.assertEqual(self.d.getVar("TEST"), "foo:val:val2:bar")
 
 class TestConcatOverride(unittest.TestCase):
     def setUp(self):
@@ -258,62 +258,66 @@
     def test_prepend(self):
         self.d.setVar("TEST", "${VAL}")
         self.d.setVar("TEST_prepend", "${FOO}:")
-        bb.data.update_data(self.d)
-        self.assertEqual(self.d.getVar("TEST", True), "foo:val")
+        self.assertEqual(self.d.getVar("TEST"), "foo:val")
 
     def test_append(self):
         self.d.setVar("TEST", "${VAL}")
         self.d.setVar("TEST_append", ":${BAR}")
-        bb.data.update_data(self.d)
-        self.assertEqual(self.d.getVar("TEST", True), "val:bar")
+        self.assertEqual(self.d.getVar("TEST"), "val:bar")
 
     def test_multiple_append(self):
         self.d.setVar("TEST", "${VAL}")
         self.d.setVar("TEST_prepend", "${FOO}:")
         self.d.setVar("TEST_append", ":val2")
         self.d.setVar("TEST_append", ":${BAR}")
-        bb.data.update_data(self.d)
-        self.assertEqual(self.d.getVar("TEST", True), "foo:val:val2:bar")
+        self.assertEqual(self.d.getVar("TEST"), "foo:val:val2:bar")
 
     def test_append_unset(self):
         self.d.setVar("TEST_prepend", "${FOO}:")
         self.d.setVar("TEST_append", ":val2")
         self.d.setVar("TEST_append", ":${BAR}")
-        bb.data.update_data(self.d)
-        self.assertEqual(self.d.getVar("TEST", True), "foo::val2:bar")
+        self.assertEqual(self.d.getVar("TEST"), "foo::val2:bar")
 
     def test_remove(self):
         self.d.setVar("TEST", "${VAL} ${BAR}")
         self.d.setVar("TEST_remove", "val")
-        bb.data.update_data(self.d)
-        self.assertEqual(self.d.getVar("TEST", True), "bar")
+        self.assertEqual(self.d.getVar("TEST"), "bar")
+
+    def test_remove_cleared(self):
+        self.d.setVar("TEST", "${VAL} ${BAR}")
+        self.d.setVar("TEST_remove", "val")
+        self.d.setVar("TEST", "${VAL} ${BAR}")
+        self.assertEqual(self.d.getVar("TEST"), "val bar")
+
+    # Ensure the value is unchanged if we have an inactive remove override
+    # (including that whitespace is preserved)
+    def test_remove_inactive_override(self):
+        self.d.setVar("TEST", "${VAL} ${BAR}    123")
+        self.d.setVar("TEST_remove_inactiveoverride", "val")
+        self.assertEqual(self.d.getVar("TEST"), "val bar    123")
 
     def test_doubleref_remove(self):
         self.d.setVar("TEST", "${VAL} ${BAR}")
         self.d.setVar("TEST_remove", "val")
         self.d.setVar("TEST_TEST", "${TEST} ${TEST}")
-        bb.data.update_data(self.d)
-        self.assertEqual(self.d.getVar("TEST_TEST", True), "bar bar")
+        self.assertEqual(self.d.getVar("TEST_TEST"), "bar bar")
 
     def test_empty_remove(self):
         self.d.setVar("TEST", "")
         self.d.setVar("TEST_remove", "val")
-        bb.data.update_data(self.d)
-        self.assertEqual(self.d.getVar("TEST", True), "")
+        self.assertEqual(self.d.getVar("TEST"), "")
 
     def test_remove_expansion(self):
         self.d.setVar("BAR", "Z")
         self.d.setVar("TEST", "${BAR}/X Y")
         self.d.setVar("TEST_remove", "${BAR}/X")
-        bb.data.update_data(self.d)
-        self.assertEqual(self.d.getVar("TEST", True), "Y")
+        self.assertEqual(self.d.getVar("TEST"), "Y")
 
     def test_remove_expansion_items(self):
         self.d.setVar("TEST", "A B C D")
         self.d.setVar("BAR", "B D")
         self.d.setVar("TEST_remove", "${BAR}")
-        bb.data.update_data(self.d)
-        self.assertEqual(self.d.getVar("TEST", True), "A C")
+        self.assertEqual(self.d.getVar("TEST"), "A C")
 
 class TestOverrides(unittest.TestCase):
     def setUp(self):
@@ -322,60 +326,53 @@
         self.d.setVar("TEST", "testvalue")
 
     def test_no_override(self):
-        bb.data.update_data(self.d)
-        self.assertEqual(self.d.getVar("TEST", True), "testvalue")
+        self.assertEqual(self.d.getVar("TEST"), "testvalue")
 
     def test_one_override(self):
         self.d.setVar("TEST_bar", "testvalue2")
-        bb.data.update_data(self.d)
-        self.assertEqual(self.d.getVar("TEST", True), "testvalue2")
+        self.assertEqual(self.d.getVar("TEST"), "testvalue2")
 
     def test_one_override_unset(self):
         self.d.setVar("TEST2_bar", "testvalue2")
-        bb.data.update_data(self.d)
-        self.assertEqual(self.d.getVar("TEST2", True), "testvalue2")
+
+        self.assertEqual(self.d.getVar("TEST2"), "testvalue2")
         self.assertCountEqual(list(self.d.keys()), ['TEST', 'TEST2', 'OVERRIDES', 'TEST2_bar'])
 
     def test_multiple_override(self):
         self.d.setVar("TEST_bar", "testvalue2")
         self.d.setVar("TEST_local", "testvalue3")
         self.d.setVar("TEST_foo", "testvalue4")
-        bb.data.update_data(self.d)
-        self.assertEqual(self.d.getVar("TEST", True), "testvalue3")
+        self.assertEqual(self.d.getVar("TEST"), "testvalue3")
         self.assertCountEqual(list(self.d.keys()), ['TEST', 'TEST_foo', 'OVERRIDES', 'TEST_bar', 'TEST_local'])
 
     def test_multiple_combined_overrides(self):
         self.d.setVar("TEST_local_foo_bar", "testvalue3")
-        bb.data.update_data(self.d)
-        self.assertEqual(self.d.getVar("TEST", True), "testvalue3")
+        self.assertEqual(self.d.getVar("TEST"), "testvalue3")
 
     def test_multiple_overrides_unset(self):
         self.d.setVar("TEST2_local_foo_bar", "testvalue3")
-        bb.data.update_data(self.d)
-        self.assertEqual(self.d.getVar("TEST2", True), "testvalue3")
+        self.assertEqual(self.d.getVar("TEST2"), "testvalue3")
 
     def test_keyexpansion_override(self):
         self.d.setVar("LOCAL", "local")
         self.d.setVar("TEST_bar", "testvalue2")
         self.d.setVar("TEST_${LOCAL}", "testvalue3")
         self.d.setVar("TEST_foo", "testvalue4")
-        bb.data.update_data(self.d)
         bb.data.expandKeys(self.d)
-        self.assertEqual(self.d.getVar("TEST", True), "testvalue3")
+        self.assertEqual(self.d.getVar("TEST"), "testvalue3")
 
     def test_rename_override(self):
         self.d.setVar("ALTERNATIVE_ncurses-tools_class-target", "a")
         self.d.setVar("OVERRIDES", "class-target")
-        bb.data.update_data(self.d)
         self.d.renameVar("ALTERNATIVE_ncurses-tools", "ALTERNATIVE_lib32-ncurses-tools")
-        self.assertEqual(self.d.getVar("ALTERNATIVE_lib32-ncurses-tools", True), "a")
+        self.assertEqual(self.d.getVar("ALTERNATIVE_lib32-ncurses-tools"), "a")
 
     def test_underscore_override(self):
         self.d.setVar("TEST_bar", "testvalue2")
         self.d.setVar("TEST_some_val", "testvalue3")
         self.d.setVar("TEST_foo", "testvalue4")
         self.d.setVar("OVERRIDES", "foo:bar:some_val")
-        self.assertEqual(self.d.getVar("TEST", True), "testvalue3")
+        self.assertEqual(self.d.getVar("TEST"), "testvalue3")
 
 class TestKeyExpansion(unittest.TestCase):
     def setUp(self):
@@ -389,7 +386,7 @@
         with LogRecord() as logs:
             bb.data.expandKeys(self.d)
             self.assertTrue(logContains("Variable key VAL_${FOO} (A) replaces original key VAL_foo (B)", logs))
-        self.assertEqual(self.d.getVar("VAL_foo", True), "A")
+        self.assertEqual(self.d.getVar("VAL_foo"), "A")
 
 class TestFlags(unittest.TestCase):
     def setUp(self):
@@ -444,3 +441,167 @@
 
         self.assertFalse(bb.utils.contains_any("SOMEFLAG", "x", True, False, self.d))
         self.assertFalse(bb.utils.contains_any("SOMEFLAG", "x y z", True, False, self.d))
+
+
+class Serialize(unittest.TestCase):
+
+    def test_serialize(self):
+        import tempfile
+        import pickle
+        d = bb.data.init()
+        d.enableTracking()
+        d.setVar('HELLO', 'world')
+        d.setVarFlag('HELLO', 'other', 'planet')
+        with tempfile.NamedTemporaryFile(delete=False) as tmpfile:
+            tmpfilename = tmpfile.name
+            pickle.dump(d, tmpfile)
+
+        with open(tmpfilename, 'rb') as f:
+            newd = pickle.load(f)
+
+        os.remove(tmpfilename)
+
+        self.assertEqual(d, newd)
+        self.assertEqual(newd.getVar('HELLO'), 'world')
+        self.assertEqual(newd.getVarFlag('HELLO', 'other'), 'planet')
+
+
+# Remote datastore tests
+# These really only test the interface, since in actual usage we have a
+# tinfoil connector that does everything over RPC, and this doesn't test
+# that.
+
+class TestConnector:
+    d = None
+    def __init__(self, d):
+        self.d = d
+    def getVar(self, name):
+        return self.d._findVar(name)
+    def getKeys(self):
+        return set(self.d.keys())
+    def getVarHistory(self, name):
+        return self.d.varhistory.variable(name)
+    def expandPythonRef(self, varname, expr, d):
+        localdata = self.d.createCopy()
+        for key in d.localkeys():
+            localdata.setVar(d.getVar(key))
+        varparse = bb.data_smart.VariableParse(varname, localdata)
+        return varparse.python_sub(expr)
+    def setVar(self, name, value):
+        self.d.setVar(name, value)
+    def setVarFlag(self, name, flag, value):
+        self.d.setVarFlag(name, flag, value)
+    def delVar(self, name):
+        self.d.delVar(name)
+        return False
+    def delVarFlag(self, name, flag):
+        self.d.delVarFlag(name, flag)
+        return False
+    def renameVar(self, name, newname):
+        self.d.renameVar(name, newname)
+        return False
+
+class Remote(unittest.TestCase):
+    def test_remote(self):
+
+        d1 = bb.data.init()
+        d1.enableTracking()
+        d2 = bb.data.init()
+        d2.enableTracking()
+        connector = TestConnector(d1)
+
+        d2.setVar('_remote_data', connector)
+
+        d1.setVar('HELLO', 'world')
+        d1.setVarFlag('OTHER', 'flagname', 'flagvalue')
+        self.assertEqual(d2.getVar('HELLO'), 'world')
+        self.assertEqual(d2.expand('${HELLO}'), 'world')
+        self.assertEqual(d2.expand('${@d.getVar("HELLO")}'), 'world')
+        self.assertIn('flagname', d2.getVarFlags('OTHER'))
+        self.assertEqual(d2.getVarFlag('OTHER', 'flagname'), 'flagvalue')
+        self.assertEqual(d1.varhistory.variable('HELLO'), d2.varhistory.variable('HELLO'))
+        # Test setVar on client side affects server
+        d2.setVar('HELLO', 'other-world')
+        self.assertEqual(d1.getVar('HELLO'), 'other-world')
+        # Test setVarFlag on client side affects server
+        d2.setVarFlag('HELLO', 'flagname', 'flagvalue')
+        self.assertEqual(d1.getVarFlag('HELLO', 'flagname'), 'flagvalue')
+        # Test client side data is incorporated in python expansion (which is done on server)
+        d2.setVar('FOO', 'bar')
+        self.assertEqual(d2.expand('${@d.getVar("FOO")}'), 'bar')
+        # Test overrides work
+        d1.setVar('FOO_test', 'baz')
+        d1.appendVar('OVERRIDES', ':test')
+        self.assertEqual(d2.getVar('FOO'), 'baz')
+
+
+# Remote equivalents of local test classes
+# Note that these aren't perfect since we only test in one direction
+
+class RemoteDataExpansions(DataExpansions):
+    def setUp(self):
+        self.d1 = bb.data.init()
+        self.d = bb.data.init()
+        self.d1["foo"] = "value_of_foo"
+        self.d1["bar"] = "value_of_bar"
+        self.d1["value_of_foo"] = "value_of_'value_of_foo'"
+        connector = TestConnector(self.d1)
+        self.d.setVar('_remote_data', connector)
+
+class TestRemoteNestedExpansions(TestNestedExpansions):
+    def setUp(self):
+        self.d1 = bb.data.init()
+        self.d = bb.data.init()
+        self.d1["foo"] = "foo"
+        self.d1["bar"] = "bar"
+        self.d1["value_of_foobar"] = "187"
+        connector = TestConnector(self.d1)
+        self.d.setVar('_remote_data', connector)
+
+class TestRemoteConcat(TestConcat):
+    def setUp(self):
+        self.d1 = bb.data.init()
+        self.d = bb.data.init()
+        self.d1.setVar("FOO", "foo")
+        self.d1.setVar("VAL", "val")
+        self.d1.setVar("BAR", "bar")
+        connector = TestConnector(self.d1)
+        self.d.setVar('_remote_data', connector)
+
+class TestRemoteConcatOverride(TestConcatOverride):
+    def setUp(self):
+        self.d1 = bb.data.init()
+        self.d = bb.data.init()
+        self.d1.setVar("FOO", "foo")
+        self.d1.setVar("VAL", "val")
+        self.d1.setVar("BAR", "bar")
+        connector = TestConnector(self.d1)
+        self.d.setVar('_remote_data', connector)
+
+class TestRemoteOverrides(TestOverrides):
+    def setUp(self):
+        self.d1 = bb.data.init()
+        self.d = bb.data.init()
+        self.d1.setVar("OVERRIDES", "foo:bar:local")
+        self.d1.setVar("TEST", "testvalue")
+        connector = TestConnector(self.d1)
+        self.d.setVar('_remote_data', connector)
+
+class TestRemoteKeyExpansion(TestKeyExpansion):
+    def setUp(self):
+        self.d1 = bb.data.init()
+        self.d = bb.data.init()
+        self.d1.setVar("FOO", "foo")
+        self.d1.setVar("BAR", "foo")
+        connector = TestConnector(self.d1)
+        self.d.setVar('_remote_data', connector)
+
+class TestRemoteFlags(TestFlags):
+    def setUp(self):
+        self.d1 = bb.data.init()
+        self.d = bb.data.init()
+        self.d1.setVar("foo", "value of foo")
+        self.d1.setVarFlag("foo", "flag1", "value of flag1")
+        self.d1.setVarFlag("foo", "flag2", "value of flag2")
+        connector = TestConnector(self.d1)
+        self.d.setVar('_remote_data', connector)
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/tests/fetch.py b/import-layers/yocto-poky/bitbake/lib/bb/tests/fetch.py
index 0fd2c02..5a8d892 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/tests/fetch.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/tests/fetch.py
@@ -793,6 +793,7 @@
                 ud = bb.fetch2.FetchData(k[1], self.d)
                 pupver= ud.method.latest_versionstring(ud, self.d)
                 verstring = pupver[0]
+                self.assertTrue(verstring, msg="Could not find upstream version")
                 r = bb.utils.vercmp_string(v, verstring)
                 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring))
 
@@ -804,6 +805,7 @@
                 ud = bb.fetch2.FetchData(k[1], self.d)
                 pupver = ud.method.latest_versionstring(ud, self.d)
                 verstring = pupver[0]
+                self.assertTrue(verstring, msg="Could not find upstream version")
                 r = bb.utils.vercmp_string(v, verstring)
                 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring))
 
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/tests/parse.py b/import-layers/yocto-poky/bitbake/lib/bb/tests/parse.py
index 0b2706a..ab6ca90 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/tests/parse.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/tests/parse.py
@@ -58,9 +58,9 @@
     def test_parse_simple(self):
         f = self.parsehelper(self.testfile)
         d = bb.parse.handle(f.name, self.d)['']
-        self.assertEqual(d.getVar("A", True), "1")
-        self.assertEqual(d.getVar("B", True), "2")
-        self.assertEqual(d.getVar("C", True), "3")
+        self.assertEqual(d.getVar("A"), "1")
+        self.assertEqual(d.getVar("B"), "2")
+        self.assertEqual(d.getVar("C"), "3")
 
     def test_parse_incomplete_function(self):
         testfileB = self.testfile.replace("}", "")
@@ -80,9 +80,9 @@
     def test_parse_unset(self):
         f = self.parsehelper(self.unsettest)
         d = bb.parse.handle(f.name, self.d)['']
-        self.assertEqual(d.getVar("A", True), None)
-        self.assertEqual(d.getVarFlag("A","flag", True), None)
-        self.assertEqual(d.getVar("B", True), "2")
+        self.assertEqual(d.getVar("A"), None)
+        self.assertEqual(d.getVarFlag("A","flag"), None)
+        self.assertEqual(d.getVar("B"), "2")
         
 
     overridetest = """
@@ -95,11 +95,11 @@
     def test_parse_overrides(self):
         f = self.parsehelper(self.overridetest)
         d = bb.parse.handle(f.name, self.d)['']
-        self.assertEqual(d.getVar("RRECOMMENDS", True), "b")
+        self.assertEqual(d.getVar("RRECOMMENDS"), "b")
         bb.data.expandKeys(d)
-        self.assertEqual(d.getVar("RRECOMMENDS", True), "b")
+        self.assertEqual(d.getVar("RRECOMMENDS"), "b")
         d.setVar("RRECOMMENDS_gtk+", "c")
-        self.assertEqual(d.getVar("RRECOMMENDS", True), "c")
+        self.assertEqual(d.getVar("RRECOMMENDS"), "c")
 
     overridetest2 = """
 EXTRA_OECONF = ""
@@ -112,7 +112,7 @@
         d = bb.parse.handle(f.name, self.d)['']
         d.appendVar("EXTRA_OECONF", " d")
         d.setVar("OVERRIDES", "class-target")
-        self.assertEqual(d.getVar("EXTRA_OECONF", True), "b c d")
+        self.assertEqual(d.getVar("EXTRA_OECONF"), "b c d")
 
     overridetest3 = """
 DESCRIPTION = "A"
@@ -124,11 +124,11 @@
         f = self.parsehelper(self.overridetest3)
         d = bb.parse.handle(f.name, self.d)['']
         bb.data.expandKeys(d)
-        self.assertEqual(d.getVar("DESCRIPTION_bc-dev", True), "A B")
+        self.assertEqual(d.getVar("DESCRIPTION_bc-dev"), "A B")
         d.setVar("DESCRIPTION", "E")
         d.setVar("DESCRIPTION_bc-dev", "C D")
         d.setVar("OVERRIDES", "bc-dev")
-        self.assertEqual(d.getVar("DESCRIPTION", True), "C D")
+        self.assertEqual(d.getVar("DESCRIPTION"), "C D")
 
 
     classextend = """
@@ -159,6 +159,6 @@
         alldata = bb.parse.handle(f.name, self.d)
         d1 = alldata['']
         d2 = alldata[cls.name]
-        self.assertEqual(d1.getVar("VAR_var", True), "B")
-        self.assertEqual(d2.getVar("VAR_var", True), None)
+        self.assertEqual(d1.getVar("VAR_var"), "B")
+        self.assertEqual(d2.getVar("VAR_var"), None)
 
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/tinfoil.py b/import-layers/yocto-poky/bitbake/lib/bb/tinfoil.py
index 9fa5b5b..928333a 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/tinfoil.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/tinfoil.py
@@ -1,6 +1,6 @@
 # tinfoil: a simple wrapper around cooker for bitbake-based command-line utilities
 #
-# Copyright (C) 2012 Intel Corporation
+# Copyright (C) 2012-2017 Intel Corporation
 # Copyright (C) 2011 Mentor Graphics Corporation
 #
 # This program is free software; you can redistribute it and/or modify
@@ -17,50 +17,210 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 import logging
-import warnings
 import os
 import sys
+import atexit
+import re
+from collections import OrderedDict, defaultdict
 
 import bb.cache
 import bb.cooker
 import bb.providers
+import bb.taskdata
 import bb.utils
-from bb.cooker import state, BBCooker, CookerFeatures
+import bb.command
+import bb.remotedata
 from bb.cookerdata import CookerConfiguration, ConfigParameters
+from bb.main import setup_bitbake, BitBakeConfigParameters, BBMainException
 import bb.fetch2
 
+
+# We need this in order to shut down the connection to the bitbake server,
+# otherwise the process will never properly exit
+_server_connections = []
+def _terminate_connections():
+    for connection in _server_connections:
+        connection.terminate()
+atexit.register(_terminate_connections)
+
+class TinfoilUIException(Exception):
+    """Exception raised when the UI returns non-zero from its main function"""
+    def __init__(self, returncode):
+        self.returncode = returncode
+    def __repr__(self):
+        return 'UI module main returned %d' % self.returncode
+
+class TinfoilCommandFailed(Exception):
+    """Exception raised when run_command fails"""
+
+class TinfoilDataStoreConnector:
+
+    def __init__(self, tinfoil, dsindex):
+        self.tinfoil = tinfoil
+        self.dsindex = dsindex
+    def getVar(self, name):
+        value = self.tinfoil.run_command('dataStoreConnectorFindVar', self.dsindex, name)
+        overrides = None
+        if isinstance(value, dict):
+            if '_connector_origtype' in value:
+                value['_content'] = self.tinfoil._reconvert_type(value['_content'], value['_connector_origtype'])
+                del value['_connector_origtype']
+            if '_connector_overrides' in value:
+                overrides = value['_connector_overrides']
+                del value['_connector_overrides']
+        return value, overrides
+    def getKeys(self):
+        return set(self.tinfoil.run_command('dataStoreConnectorGetKeys', self.dsindex))
+    def getVarHistory(self, name):
+        return self.tinfoil.run_command('dataStoreConnectorGetVarHistory', self.dsindex, name)
+    def expandPythonRef(self, varname, expr, d):
+        ds = bb.remotedata.RemoteDatastores.transmit_datastore(d)
+        ret = self.tinfoil.run_command('dataStoreConnectorExpandPythonRef', ds, varname, expr)
+        return ret
+    def setVar(self, varname, value):
+        if self.dsindex is None:
+            self.tinfoil.run_command('setVariable', varname, value)
+        else:
+            # Not currently implemented - indicate that setting should
+            # be redirected to local side
+            return True
+    def setVarFlag(self, varname, flagname, value):
+        if self.dsindex is None:
+            self.tinfoil.run_command('dataStoreConnectorSetVarFlag', self.dsindex, varname, flagname, value)
+        else:
+            # Not currently implemented - indicate that setting should
+            # be redirected to local side
+            return True
+    def delVar(self, varname):
+        if self.dsindex is None:
+            self.tinfoil.run_command('dataStoreConnectorDelVar', self.dsindex, varname)
+        else:
+            # Not currently implemented - indicate that setting should
+            # be redirected to local side
+            return True
+    def delVarFlag(self, varname, flagname):
+        if self.dsindex is None:
+            self.tinfoil.run_command('dataStoreConnectorDelVar', self.dsindex, varname, flagname)
+        else:
+            # Not currently implemented - indicate that setting should
+            # be redirected to local side
+            return True
+    def renameVar(self, name, newname):
+        if self.dsindex is None:
+            self.tinfoil.run_command('dataStoreConnectorRenameVar', self.dsindex, name, newname)
+        else:
+            # Not currently implemented - indicate that setting should
+            # be redirected to local side
+            return True
+
+class TinfoilCookerAdapter:
+    """
+    Provide an adapter for existing code that expects to access a cooker object via Tinfoil,
+    since now Tinfoil is on the client side it no longer has direct access.
+    """
+
+    class TinfoilCookerCollectionAdapter:
+        """ cooker.collection adapter """
+        def __init__(self, tinfoil):
+            self.tinfoil = tinfoil
+        def get_file_appends(self, fn):
+            return self.tinfoil.get_file_appends(fn)
+        def __getattr__(self, name):
+            if name == 'overlayed':
+                return self.tinfoil.get_overlayed_recipes()
+            elif name == 'bbappends':
+                return self.tinfoil.run_command('getAllAppends')
+            else:
+                raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
+
+    class TinfoilRecipeCacheAdapter:
+        """ cooker.recipecache adapter """
+        def __init__(self, tinfoil):
+            self.tinfoil = tinfoil
+            self._cache = {}
+
+        def get_pkg_pn_fn(self):
+            pkg_pn = defaultdict(list, self.tinfoil.run_command('getRecipes') or [])
+            pkg_fn = {}
+            for pn, fnlist in pkg_pn.items():
+                for fn in fnlist:
+                    pkg_fn[fn] = pn
+            self._cache['pkg_pn'] = pkg_pn
+            self._cache['pkg_fn'] = pkg_fn
+
+        def __getattr__(self, name):
+            # Grab these only when they are requested since they aren't always used
+            if name in self._cache:
+                return self._cache[name]
+            elif name == 'pkg_pn':
+                self.get_pkg_pn_fn()
+                return self._cache[name]
+            elif name == 'pkg_fn':
+                self.get_pkg_pn_fn()
+                return self._cache[name]
+            elif name == 'deps':
+                attrvalue = defaultdict(list, self.tinfoil.run_command('getRecipeDepends') or [])
+            elif name == 'rundeps':
+                attrvalue = defaultdict(lambda: defaultdict(list), self.tinfoil.run_command('getRuntimeDepends') or [])
+            elif name == 'runrecs':
+                attrvalue = defaultdict(lambda: defaultdict(list), self.tinfoil.run_command('getRuntimeRecommends') or [])
+            elif name == 'pkg_pepvpr':
+                attrvalue = self.tinfoil.run_command('getRecipeVersions') or {}
+            elif name == 'inherits':
+                attrvalue = self.tinfoil.run_command('getRecipeInherits') or {}
+            elif name == 'bbfile_priority':
+                attrvalue = self.tinfoil.run_command('getBbFilePriority') or {}
+            elif name == 'pkg_dp':
+                attrvalue = self.tinfoil.run_command('getDefaultPreference') or {}
+            else:
+                raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
+
+            self._cache[name] = attrvalue
+            return attrvalue
+
+    def __init__(self, tinfoil):
+        self.tinfoil = tinfoil
+        self.collection = self.TinfoilCookerCollectionAdapter(tinfoil)
+        self.recipecaches = {}
+        # FIXME all machines
+        self.recipecaches[''] = self.TinfoilRecipeCacheAdapter(tinfoil)
+        self._cache = {}
+    def __getattr__(self, name):
+        # Grab these only when they are requested since they aren't always used
+        if name in self._cache:
+            return self._cache[name]
+        elif name == 'skiplist':
+            attrvalue = self.tinfoil.get_skipped_recipes()
+        elif name == 'bbfile_config_priorities':
+            ret = self.tinfoil.run_command('getLayerPriorities')
+            bbfile_config_priorities = []
+            for collection, pattern, regex, pri in ret:
+                bbfile_config_priorities.append((collection, pattern, re.compile(regex), pri))
+
+            attrvalue = bbfile_config_priorities
+        else:
+            raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
+
+        self._cache[name] = attrvalue
+        return attrvalue
+
+    def findBestProvider(self, pn):
+        return self.tinfoil.find_best_provider(pn)
+
+
 class Tinfoil:
-    def __init__(self, output=sys.stdout, tracking=False):
-        # Needed to avoid deprecation warnings with python 2.6
-        warnings.filterwarnings("ignore", category=DeprecationWarning)
 
-        # Set up logging
+    def __init__(self, output=sys.stdout, tracking=False, setup_logging=True):
         self.logger = logging.getLogger('BitBake')
-        self._log_hdlr = logging.StreamHandler(output)
-        bb.msg.addDefaultlogFilter(self._log_hdlr)
-        format = bb.msg.BBLogFormatter("%(levelname)s: %(message)s")
-        if output.isatty():
-            format.enable_color()
-        self._log_hdlr.setFormatter(format)
-        self.logger.addHandler(self._log_hdlr)
-
-        self.config = CookerConfiguration()
-        configparams = TinfoilConfigParameters(parse_only=True)
-        self.config.setConfigParameters(configparams)
-        self.config.setServerRegIdleCallback(self.register_idle_function)
-        features = []
-        if tracking:
-            features.append(CookerFeatures.BASEDATASTORE_TRACKING)
-        cleanedvars = bb.utils.clean_environment()
-        self.cooker = BBCooker(self.config, features)
-        self.config_data = self.cooker.data
-        bb.providers.logger.setLevel(logging.ERROR)
-        self.cooker_data = None
-        for k in cleanedvars:
-            os.environ[k] = cleanedvars[k]
-
-    def register_idle_function(self, function, data):
-        pass
+        self.config_data = None
+        self.cooker = None
+        self.tracking = tracking
+        self.ui_module = None
+        self.server_connection = None
+        if setup_logging:
+            # This is the *client-side* logger, nothing to do with
+            # logging messages from the server
+            bb.msg.logger_create('BitBake', output)
 
     def __enter__(self):
         return self
@@ -68,30 +228,161 @@
     def __exit__(self, type, value, traceback):
         self.shutdown()
 
-    def parseRecipes(self):
-        sys.stderr.write("Parsing recipes..")
-        self.logger.setLevel(logging.WARNING)
+    def prepare(self, config_only=False, config_params=None, quiet=0):
+        if self.tracking:
+            extrafeatures = [bb.cooker.CookerFeatures.BASEDATASTORE_TRACKING]
+        else:
+            extrafeatures = []
 
-        try:
-            while self.cooker.state in (state.initial, state.parsing):
-                self.cooker.updateCache()
-        except KeyboardInterrupt:
-            self.cooker.shutdown()
-            self.cooker.updateCache()
-            sys.exit(2)
+        if not config_params:
+            config_params = TinfoilConfigParameters(config_only=config_only, quiet=quiet)
 
-        self.logger.setLevel(logging.INFO)
-        sys.stderr.write("done.\n")
+        cookerconfig = CookerConfiguration()
+        cookerconfig.setConfigParameters(config_params)
 
-        self.cooker_data = self.cooker.recipecaches['']
+        server, self.server_connection, ui_module = setup_bitbake(config_params,
+                            cookerconfig,
+                            extrafeatures)
 
-    def prepare(self, config_only = False):
-        if not self.cooker_data:
+        self.ui_module = ui_module
+
+        # Ensure the path to bitbake's bin directory is in PATH so that things like
+        # bitbake-worker can be run (usually this is the case, but it doesn't have to be)
+        path = os.getenv('PATH').split(':')
+        bitbakebinpath = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', 'bin'))
+        for entry in path:
+            if entry.endswith(os.sep):
+                entry = entry[:-1]
+            if os.path.abspath(entry) == bitbakebinpath:
+                break
+        else:
+            path.insert(0, bitbakebinpath)
+            os.environ['PATH'] = ':'.join(path)
+
+        if self.server_connection:
+            _server_connections.append(self.server_connection)
             if config_only:
-                self.cooker.parseConfiguration()
-                self.cooker_data = self.cooker.recipecaches['']
+                config_params.updateToServer(self.server_connection.connection, os.environ.copy())
+                self.run_command('parseConfiguration')
             else:
-                self.parseRecipes()
+                self.run_actions(config_params)
+
+            self.config_data = bb.data.init()
+            connector = TinfoilDataStoreConnector(self, None)
+            self.config_data.setVar('_remote_data', connector)
+            self.cooker = TinfoilCookerAdapter(self)
+            self.cooker_data = self.cooker.recipecaches['']
+        else:
+            raise Exception('Failed to start bitbake server')
+
+    def run_actions(self, config_params):
+        """
+        Run the actions specified in config_params through the UI.
+        """
+        ret = self.ui_module.main(self.server_connection.connection, self.server_connection.events, config_params)
+        if ret:
+            raise TinfoilUIException(ret)
+
+    def parseRecipes(self):
+        """
+        Force a parse of all recipes. Normally you should specify
+        config_only=False when calling prepare() instead of using this
+        function; this function is designed for situations where you need
+        to initialise Tinfoil and use it with config_only=True first and
+        then conditionally call this function to parse recipes later.
+        """
+        config_params = TinfoilConfigParameters(config_only=False)
+        self.run_actions(config_params)
+
+    def run_command(self, command, *params):
+        """
+        Run a command on the server (as implemented in bb.command).
+        Note that there are two types of command - synchronous and
+        asynchronous; in order to receive the results of asynchronous
+        commands you will need to set an appropriate event mask
+        using set_event_mask() and listen for the result using
+        wait_event() - with the correct event mask you'll at least get
+        bb.command.CommandCompleted and possibly other events before
+        that depending on the command.
+        """
+        if not self.server_connection:
+            raise Exception('Not connected to server (did you call .prepare()?)')
+
+        commandline = [command]
+        if params:
+            commandline.extend(params)
+        result = self.server_connection.connection.runCommand(commandline)
+        if result[1]:
+            raise TinfoilCommandFailed(result[1])
+        return result[0]
+
+    def set_event_mask(self, eventlist):
+        """Set the event mask which will be applied within wait_event()"""
+        if not self.server_connection:
+            raise Exception('Not connected to server (did you call .prepare()?)')
+        llevel, debug_domains = bb.msg.constructLogOptions()
+        ret = self.run_command('setEventMask', self.server_connection.connection.getEventHandle(), llevel, debug_domains, eventlist)
+        if not ret:
+            raise Exception('setEventMask failed')
+
+    def wait_event(self, timeout=0):
+        """
+        Wait for an event from the server for the specified time.
+        A timeout of 0 means don't wait if there are no events in the queue.
+        Returns the next event in the queue or None if the timeout was
+        reached. Note that in order to recieve any events you will
+        first need to set the internal event mask using set_event_mask()
+        (otherwise whatever event mask the UI set up will be in effect).
+        """
+        if not self.server_connection:
+            raise Exception('Not connected to server (did you call .prepare()?)')
+        return self.server_connection.events.waitEvent(timeout)
+
+    def get_overlayed_recipes(self):
+        return defaultdict(list, self.run_command('getOverlayedRecipes'))
+
+    def get_skipped_recipes(self):
+        return OrderedDict(self.run_command('getSkippedRecipes'))
+
+    def get_all_providers(self):
+        return defaultdict(list, self.run_command('allProviders'))
+
+    def find_providers(self):
+        return self.run_command('findProviders')
+
+    def find_best_provider(self, pn):
+        return self.run_command('findBestProvider', pn)
+
+    def get_runtime_providers(self, rdep):
+        return self.run_command('getRuntimeProviders', rdep)
+
+    def get_recipe_file(self, pn):
+        """
+        Get the file name for the specified recipe/target. Raises
+        bb.providers.NoProvider if there is no match or the recipe was
+        skipped.
+        """
+        best = self.find_best_provider(pn)
+        if not best or (len(best) > 3 and not best[3]):
+            skiplist = self.get_skipped_recipes()
+            taskdata = bb.taskdata.TaskData(None, skiplist=skiplist)
+            skipreasons = taskdata.get_reasons(pn)
+            if skipreasons:
+                raise bb.providers.NoProvider('%s is unavailable:\n  %s' % (pn, '  \n'.join(skipreasons)))
+            else:
+                raise bb.providers.NoProvider('Unable to find any recipe file matching "%s"' % pn)
+        return best[3]
+
+    def get_file_appends(self, fn):
+        return self.run_command('getFileAppends', fn)
+
+    def parse_recipe(self, pn):
+        """
+        Parse the specified recipe and return a datastore object
+        representing the environment for the recipe.
+        """
+        fn = self.get_recipe_file(pn)
+        return self.parse_recipe_file(fn)
 
     def parse_recipe_file(self, fn, appends=True, appendlist=None, config_data=None):
         """
@@ -110,41 +401,82 @@
         """
         if appends and appendlist == []:
             appends = False
-        if appends:
-            if appendlist:
-                appendfiles = appendlist
-            else:
-                if not hasattr(self.cooker, 'collection'):
-                    raise Exception('You must call tinfoil.prepare() with config_only=False in order to get bbappends')
-                appendfiles = self.cooker.collection.get_file_appends(fn)
-        else:
-            appendfiles = None
         if config_data:
-            # We have to use a different function here if we're passing in a datastore
-            localdata = bb.data.createCopy(config_data)
-            envdata = bb.cache.parse_recipe(localdata, fn, appendfiles)['']
+            dctr = bb.remotedata.RemoteDatastores.transmit_datastore(config_data)
+            dscon = self.run_command('parseRecipeFile', fn, appends, appendlist, dctr)
         else:
-            # Use the standard path
-            parser = bb.cache.NoCache(self.cooker.databuilder)
-            envdata = parser.loadDataFull(fn, appendfiles)
-        return envdata
+            dscon = self.run_command('parseRecipeFile', fn, appends, appendlist)
+        if dscon:
+            return self._reconvert_type(dscon, 'DataStoreConnectionHandle')
+        else:
+            return None
+
+    def build_file(self, buildfile, task):
+        """
+        Runs the specified task for just a single recipe (i.e. no dependencies).
+        This is equivalent to bitbake -b, except no warning will be printed.
+        """
+        return self.run_command('buildFile', buildfile, task, True)
 
     def shutdown(self):
-        self.cooker.shutdown(force=True)
-        self.cooker.post_serve()
-        self.cooker.unlockBitbake()
-        self.logger.removeHandler(self._log_hdlr)
+        if self.server_connection:
+            self.run_command('clientComplete')
+            _server_connections.remove(self.server_connection)
+            bb.event.ui_queue = []
+            self.server_connection.terminate()
+            self.server_connection = None
 
-class TinfoilConfigParameters(ConfigParameters):
+    def _reconvert_type(self, obj, origtypename):
+        """
+        Convert an object back to the right type, in the case
+        that marshalling has changed it (especially with xmlrpc)
+        """
+        supported_types = {
+            'set': set,
+            'DataStoreConnectionHandle': bb.command.DataStoreConnectionHandle,
+        }
 
-    def __init__(self, **options):
+        origtype = supported_types.get(origtypename, None)
+        if origtype is None:
+            raise Exception('Unsupported type "%s"' % origtypename)
+        if type(obj) == origtype:
+            newobj = obj
+        elif isinstance(obj, dict):
+            # New style class
+            newobj = origtype()
+            for k,v in obj.items():
+                setattr(newobj, k, v)
+        else:
+            # Assume we can coerce the type
+            newobj = origtype(obj)
+
+        if isinstance(newobj, bb.command.DataStoreConnectionHandle):
+            connector = TinfoilDataStoreConnector(self, newobj.dsindex)
+            newobj = bb.data.init()
+            newobj.setVar('_remote_data', connector)
+
+        return newobj
+
+
+class TinfoilConfigParameters(BitBakeConfigParameters):
+
+    def __init__(self, config_only, **options):
         self.initial_options = options
+        # Apply some sane defaults
+        if not 'parse_only' in options:
+            self.initial_options['parse_only'] = not config_only
+        #if not 'status_only' in options:
+        #    self.initial_options['status_only'] = config_only
+        if not 'ui' in options:
+            self.initial_options['ui'] = 'knotty'
+        if not 'argv' in options:
+            self.initial_options['argv'] = []
+
         super(TinfoilConfigParameters, self).__init__()
 
-    def parseCommandLine(self, argv=sys.argv):
-        class DummyOptions:
-            def __init__(self, initial_options):
-                for key, val in initial_options.items():
-                    setattr(self, key, val)
-
-        return DummyOptions(self.initial_options), None
+    def parseCommandLine(self, argv=None):
+        # We don't want any parameters parsed from the command line
+        opts = super(TinfoilConfigParameters, self).parseCommandLine([])
+        for key, val in self.initial_options.items():
+            setattr(opts[0], key, val)
+        return opts
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/buildinfohelper.py b/import-layers/yocto-poky/bitbake/lib/bb/ui/buildinfohelper.py
index 3ddcb2a..e451c63 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/ui/buildinfohelper.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/ui/buildinfohelper.py
@@ -42,10 +42,12 @@
 from orm.models import Package, Package_File, Target_Installed_Package, Target_File
 from orm.models import Task_Dependency, Package_Dependency
 from orm.models import Recipe_Dependency, Provides
-from orm.models import Project, CustomImagePackage, CustomImageRecipe
+from orm.models import Project, CustomImagePackage
 from orm.models import signal_runbuilds
 
 from bldcontrol.models import BuildEnvironment, BuildRequest
+from bldcontrol.models import BRLayer
+from bldcontrol import bbcontroller
 
 from bb.msg import BBLogFormatter as formatter
 from django.db import models
@@ -361,11 +363,6 @@
 
     def get_update_layer_version_object(self, build_obj, layer_obj, layer_version_information):
         if isinstance(layer_obj, Layer_Version):
-            # Special case the toaster-custom-images layer which is created
-            # on the fly so don't update the values which may cause the layer
-            # to be duplicated on a future get_or_create
-            if layer_obj.layer.name == CustomImageRecipe.LAYER_NAME:
-                return layer_obj
             # We already found our layer version for this build so just
             # update it with the new build information
             logger.debug("We found our layer from toaster")
@@ -384,8 +381,8 @@
                 local_path=layer_version_information['local_path'],
             )
 
-            logger.info("created new historical layer version %d",
-                        layer_copy.pk)
+            logger.debug("Created new layer version %s for build history",
+                         layer_copy.layer.name)
 
             self.layer_version_built.append(layer_copy)
 
@@ -441,48 +438,33 @@
         else:
             br_id, be_id = brbe.split(":")
 
-            # find layer by checkout path;
-            from bldcontrol import bbcontroller
-            bc = bbcontroller.getBuildEnvironmentController(pk = be_id)
+            # Find the layer version by matching the layer event information
+            # against the metadata we have in Toaster
 
-            # we might have a race condition here, as the project layers may change between the build trigger and the actual build execution
-            # but we can only match on the layer name, so the worst thing can happen is a mis-identification of the layer, not a total failure
+            try:
+                br_layer = BRLayer.objects.get(req=br_id,
+                                               name=layer_information['name'])
+                return br_layer.layer_version
+            except (BRLayer.MultipleObjectsReturned, BRLayer.DoesNotExist):
+                # There are multiple of the same layer name or the name
+                # hasn't been determined by the toaster.bbclass layer
+                # so let's filter by the local_path
+                bc = bbcontroller.getBuildEnvironmentController(pk=be_id)
+                for br_layer in BRLayer.objects.filter(req=br_id):
+                    if br_layer.giturl and \
+                       layer_information['local_path'].endswith(
+                           bc.getGitCloneDirectory(br_layer.giturl,
+                                                   br_layer.commit)):
+                            return br_layer.layer_version
 
-            # note that this is different
-            buildrequest = BuildRequest.objects.get(pk = br_id)
-            for brl in buildrequest.brlayer_set.all():
-                if brl.local_source_dir:
-                    localdirname = os.path.join(brl.local_source_dir,
-                                                brl.dirpath)
-                else:
-                    localdirname = os.path.join(bc.getGitCloneDirectory(brl.giturl, brl.commit), brl.dirpath)
-                # we get a relative path, unless running in HEAD mode where the path is absolute
-                if not localdirname.startswith("/"):
-                    localdirname = os.path.join(bc.be.sourcedir, localdirname)
-                #logger.debug(1, "Localdirname %s lcal_path %s" % (localdirname, layer_information['local_path']))
-                if localdirname.startswith(layer_information['local_path']):
-                  # If the build request came from toaster this field
-                  # should contain the information from the layer_version
-                  # That created this build request.
-                    if brl.layer_version:
-                        return brl.layer_version
+                    if br_layer.local_source_dir == \
+                            layer_information['local_path']:
+                        return br_layer.layer_version
 
-                # This might be a local layer (i.e. no git info) so try
-                # matching local_source_dir
-                if brl.local_source_dir and brl.local_source_dir == layer_information["local_path"]:
-                    return brl.layer_version
-
-                    # we matched the BRLayer, but we need the layer_version that generated this BR; reverse of the Project.schedule_build()
-                    #logger.debug(1, "Matched %s to BRlayer %s" % (pformat(layer_information["local_path"]), localdirname))
-
-                    for pl in buildrequest.project.projectlayer_set.filter(layercommit__layer__name = brl.name):
-                        if pl.layercommit.layer.vcs_url == brl.giturl :
-                            layer = pl.layercommit.layer
-                            layer.save()
-                            return layer
-
-            raise NotExisting("Unidentified layer %s" % pformat(layer_information))
-
+        # We've reached the end of our search and couldn't find the layer
+        # we can continue but some data may be missing
+        raise NotExisting("Unidentified layer %s" %
+                          pformat(layer_information))
 
     def save_target_file_information(self, build_obj, target_obj, filedata):
         assert isinstance(build_obj, Build)
@@ -876,6 +858,12 @@
         self.pathname = None
         self.lineno = None
 
+    def getMessage(self):
+        """
+        Simulate LogRecord message return
+        """
+        return self.msg
+
 
 class BuildInfoHelper(object):
     """ This class gathers the build information from the server and sends it
@@ -983,9 +971,10 @@
         return task_information
 
     def _get_layer_version_for_dependency(self, pathRE):
-        """ Returns the layer in the toaster db that has a full regex match to the pathRE.
-        pathRE - the layer path passed as a regex in the event. It is created in
-          cooker.py as a collection for the layer priorities.
+        """ Returns the layer in the toaster db that has a full regex
+        match to the pathRE. pathRE - the layer path passed as a regex in the
+        event. It is created in cooker.py as a collection for the layer
+        priorities.
         """
         self._ensure_build()
 
@@ -993,19 +982,31 @@
             assert isinstance(layer_version, Layer_Version)
             return len(layer_version.local_path)
 
-        # we don't care if we match the trailing slashes
-        p = re.compile(re.sub("/[^/]*?$","",pathRE))
-        # Heuristics: we always match recipe to the deepest layer path in the discovered layers
-        for lvo in sorted(self.orm_wrapper.layer_version_objects, reverse=True, key=_sort_longest_path):
-            if p.fullmatch(lvo.local_path):
+        # Our paths don't append a trailing slash
+        if pathRE.endswith("/"):
+            pathRE = pathRE[:-1]
+
+        p = re.compile(pathRE)
+        path=re.sub(r'[$^]',r'',pathRE)
+        # Heuristics: we always match recipe to the deepest layer path in
+        # the discovered layers
+        for lvo in sorted(self.orm_wrapper.layer_version_objects,
+                          reverse=True, key=_sort_longest_path):
+            if p.fullmatch(os.path.abspath(lvo.local_path)):
                 return lvo
             if lvo.layer.local_source_dir:
-                if p.fullmatch(lvo.layer.local_source_dir):
+                if p.fullmatch(os.path.abspath(lvo.layer.local_source_dir)):
                     return lvo
-        #if we get here, we didn't read layers correctly; dump whatever information we have on the error log
-        logger.warning("Could not match layer dependency for path %s : %s", path, self.orm_wrapper.layer_version_objects)
+            if 0 == path.find(lvo.local_path):
+                # sub-layer path inside existing layer
+                return lvo
 
-
+        # if we get here, we didn't read layers correctly;
+        # dump whatever information we have on the error log
+        logger.warning("Could not match layer dependency for path %s : %s",
+                       pathRE,
+                       self.orm_wrapper.layer_version_objects)
+        return None
 
     def _get_layer_version_for_path(self, path):
         self._ensure_build()
@@ -1268,6 +1269,14 @@
                 candidates = [x for x in self.internal_state['taskdata'].keys() if x.endswith(identifier)]
                 if len(candidates) == 1:
                     identifier = candidates[0]
+                elif len(candidates) > 1 and hasattr(event,'_package'):
+                    if 'native-' in event._package:
+                        identifier = 'native:' + identifier
+                    if 'nativesdk-' in event._package:
+                        identifier = 'nativesdk:' + identifier
+                    candidates = [x for x in self.internal_state['taskdata'].keys() if x.endswith(identifier)]
+                    if len(candidates) == 1:
+                        identifier = candidates[0]
 
         assert identifier in self.internal_state['taskdata']
         identifierlist = identifier.split(":")
@@ -1398,9 +1407,9 @@
             for lv in event._depgraph['layer-priorities']:
                 (_, path, _, priority) = lv
                 layer_version_obj = self._get_layer_version_for_dependency(path)
-                assert layer_version_obj is not None
-                layer_version_obj.priority = priority
-                layer_version_obj.save()
+                if layer_version_obj:
+                    layer_version_obj.priority = priority
+                    layer_version_obj.save()
 
         # save recipe information
         self.internal_state['recipes'] = {}
@@ -1665,6 +1674,36 @@
                 break
         return endswith
 
+    def scan_task_artifacts(self, event):
+        """
+        The 'TaskArtifacts' event passes the manifest file content for the
+        tasks 'do_deploy', 'do_image_complete', 'do_populate_sdk', and
+        'do_populate_sdk_ext'. The first two will be implemented later.
+        """
+        task_vars = BuildInfoHelper._get_data_from_event(event)
+        task_name = task_vars['task'][task_vars['task'].find(':')+1:]
+        task_artifacts = task_vars['artifacts']
+
+        if task_name in ['do_populate_sdk', 'do_populate_sdk_ext']:
+            targets = [target for target in self.internal_state['targets'] \
+                if target.task == task_name[3:]]
+            if not targets:
+                logger.warning("scan_task_artifacts: SDK targets not found: %s\n", task_name)
+                return
+            for artifact_path in task_artifacts:
+                if not os.path.isfile(artifact_path):
+                    logger.warning("scan_task_artifacts: artifact file not found: %s\n", artifact_path)
+                    continue
+                for target in targets:
+                    # don't record the file if it's already been added
+                    # to this target
+                    matching_files = TargetSDKFile.objects.filter(
+                        target=target, file_name=artifact_path)
+                    if matching_files.count() == 0:
+                        artifact_size = os.stat(artifact_path).st_size
+                        self.orm_wrapper.save_target_sdk_file(
+                            target, artifact_path, artifact_size)
+
     def _get_image_files(self, deploy_dir_image, image_name, image_file_extensions):
         """
         Find files in deploy_dir_image whose basename starts with the
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/knotty.py b/import-layers/yocto-poky/bitbake/lib/bb/ui/knotty.py
index 948f527..82aa7c4 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/ui/knotty.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/ui/knotty.py
@@ -75,10 +75,8 @@
                 extrastr = str(extra)
                 if extrastr[0] != ' ':
                     extrastr = ' ' + extrastr
-                if extrastr[-1] != ' ':
-                    extrastr += ' '
             else:
-                extrastr = ' '
+                extrastr = ''
             self.widgets[self.extrapos] = extrastr
 
     def _need_update(self):
@@ -284,7 +282,7 @@
             content = self.main_progress.update(progress)
             print('')
         lines = 1 + int(len(content) / (self.columns + 1))
-        if not self.quiet:
+        if self.quiet == 0:
             for tasknum, task in enumerate(tasks[:(self.rows - 2)]):
                 if isinstance(task, tuple):
                     pbar, progress, rate, start_time = task
@@ -312,7 +310,7 @@
             fd = sys.stdin.fileno()
             self.termios.tcsetattr(fd, self.termios.TCSADRAIN, self.stdinbackup)
 
-def _log_settings_from_server(server):
+def _log_settings_from_server(server, observe_only):
     # Get values of variables which control our output
     includelogs, error = server.runCommand(["getVariable", "BBINCLUDELOGS"])
     if error:
@@ -322,7 +320,11 @@
     if error:
         logger.error("Unable to get the value of BBINCLUDELOGS_LINES variable: %s" % error)
         raise BaseException(error)
-    consolelogfile, error = server.runCommand(["getSetVariable", "BB_CONSOLELOG"])
+    if observe_only:
+        cmd = 'getVariable'
+    else:
+        cmd = 'getSetVariable'
+    consolelogfile, error = server.runCommand([cmd, "BB_CONSOLELOG"])
     if error:
         logger.error("Unable to get the value of BB_CONSOLELOG variable: %s" % error)
         raise BaseException(error)
@@ -340,7 +342,7 @@
 
 def main(server, eventHandler, params, tf = TerminalFilter):
 
-    includelogs, loglines, consolelogfile = _log_settings_from_server(server)
+    includelogs, loglines, consolelogfile = _log_settings_from_server(server, params.observe_only)
 
     if sys.stdin.isatty() and sys.stdout.isatty():
         log_exec_tty = True
@@ -353,10 +355,13 @@
     errconsole = logging.StreamHandler(sys.stderr)
     format_str = "%(levelname)s: %(message)s"
     format = bb.msg.BBLogFormatter(format_str)
-    if params.options.quiet:
-        bb.msg.addDefaultlogFilter(console, bb.msg.BBLogFilterStdOut, bb.msg.BBLogFormatter.WARNING)
+    if params.options.quiet == 0:
+        forcelevel = None
+    elif params.options.quiet > 2:
+        forcelevel = bb.msg.BBLogFormatter.ERROR
     else:
-        bb.msg.addDefaultlogFilter(console, bb.msg.BBLogFilterStdOut)
+        forcelevel = bb.msg.BBLogFormatter.WARNING
+    bb.msg.addDefaultlogFilter(console, bb.msg.BBLogFilterStdOut, forcelevel)
     bb.msg.addDefaultlogFilter(errconsole, bb.msg.BBLogFilterStdErr)
     console.setFormatter(format)
     errconsole.setFormatter(format)
@@ -506,35 +511,47 @@
                 logger.info(event._message)
                 continue
             if isinstance(event, bb.event.ParseStarted):
+                if params.options.quiet > 1:
+                    continue
                 if event.total == 0:
                     continue
                 parseprogress = new_progress("Parsing recipes", event.total).start()
                 continue
             if isinstance(event, bb.event.ParseProgress):
+                if params.options.quiet > 1:
+                    continue
                 if parseprogress:
                     parseprogress.update(event.current)
                 else:
                     bb.warn("Got ParseProgress event for parsing that never started?")
                 continue
             if isinstance(event, bb.event.ParseCompleted):
+                if params.options.quiet > 1:
+                    continue
                 if not parseprogress:
                     continue
                 parseprogress.finish()
                 pasreprogress = None
-                if not params.options.quiet:
+                if params.options.quiet == 0:
                     print(("Parsing of %d .bb files complete (%d cached, %d parsed). %d targets, %d skipped, %d masked, %d errors."
                         % ( event.total, event.cached, event.parsed, event.virtuals, event.skipped, event.masked, event.errors)))
                 continue
 
             if isinstance(event, bb.event.CacheLoadStarted):
+                if params.options.quiet > 1:
+                    continue
                 cacheprogress = new_progress("Loading cache", event.total).start()
                 continue
             if isinstance(event, bb.event.CacheLoadProgress):
+                if params.options.quiet > 1:
+                    continue
                 cacheprogress.update(event.current)
                 continue
             if isinstance(event, bb.event.CacheLoadCompleted):
+                if params.options.quiet > 1:
+                    continue
                 cacheprogress.finish()
-                if not params.options.quiet:
+                if params.options.quiet == 0:
                     print("Loaded %d entries from dependency cache." % event.num_entries)
                 continue
 
@@ -620,16 +637,22 @@
                 continue
 
             if isinstance(event, bb.event.ProcessStarted):
+                if params.options.quiet > 1:
+                    continue
                 parseprogress = new_progress(event.processname, event.total)
                 parseprogress.start(False)
                 continue
             if isinstance(event, bb.event.ProcessProgress):
+                if params.options.quiet > 1:
+                    continue
                 if parseprogress:
                     parseprogress.update(event.progress)
                 else:
                     bb.warn("Got ProcessProgress event for someting that never started?")
                 continue
             if isinstance(event, bb.event.ProcessFinished):
+                if params.options.quiet > 1:
+                    continue
                 if parseprogress:
                     parseprogress.finish()
                 parseprogress = None
@@ -647,6 +670,7 @@
                                   bb.event.OperationCompleted,
                                   bb.event.OperationProgress,
                                   bb.event.DiskFull,
+                                  bb.event.HeartbeatEvent,
                                   bb.build.TaskProgress)):
                 continue
 
@@ -700,7 +724,7 @@
         if return_value and errors:
             summary += pluralise("\nSummary: There was %s ERROR message shown, returning a non-zero exit code.",
                                  "\nSummary: There were %s ERROR messages shown, returning a non-zero exit code.", errors)
-        if summary and not params.options.quiet:
+        if summary and params.options.quiet == 0:
             print(summary)
 
         if interrupted:
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/ncurses.py b/import-layers/yocto-poky/bitbake/lib/bb/ui/ncurses.py
index d81e413..ca845a3 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/ui/ncurses.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/ui/ncurses.py
@@ -297,7 +297,7 @@
 #                            bb.error("log data follows (%s)" % logfile)
 #                            number_of_lines = data.getVar("BBINCLUDELOGS_LINES", d)
 #                            if number_of_lines:
-#                                subprocess.call('tail -n%s %s' % (number_of_lines, logfile), shell=True)
+#                                subprocess.check_call('tail -n%s %s' % (number_of_lines, logfile), shell=True)
 #                            else:
 #                                f = open(logfile, "r")
 #                                while True:
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/depexp.py b/import-layers/yocto-poky/bitbake/lib/bb/ui/taskexp.py
similarity index 90%
rename from import-layers/yocto-poky/bitbake/lib/bb/ui/depexp.py
rename to import-layers/yocto-poky/bitbake/lib/bb/ui/taskexp.py
index d879e04..9d14ece 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/ui/depexp.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/ui/taskexp.py
@@ -78,7 +78,7 @@
 class DepExplorer(Gtk.Window):
     def __init__(self):
         Gtk.Window.__init__(self)
-        self.set_title("Dependency Explorer")
+        self.set_title("Task Dependency Explorer")
         self.set_default_size(500, 500)
         self.connect("delete-event", Gtk.main_quit)
 
@@ -106,30 +106,21 @@
 
         box = Gtk.VBox(homogeneous=True, spacing=4)
 
-        # Runtime Depends
+        # Task Depends
         scrolled = Gtk.ScrolledWindow()
         scrolled.set_policy(Gtk.PolicyType.AUTOMATIC, Gtk.PolicyType.AUTOMATIC)
         scrolled.set_shadow_type(Gtk.ShadowType.IN)
-        self.rdep_treeview = PackageDepView(self.depends_model, TYPE_RDEP, "Runtime Depends")
-        self.rdep_treeview.connect("row-activated", self.on_package_activated, COL_DEP_PACKAGE)
-        scrolled.add(self.rdep_treeview)
-        box.add(scrolled)
-
-        # Build Depends
-        scrolled = Gtk.ScrolledWindow()
-        scrolled.set_policy(Gtk.PolicyType.AUTOMATIC, Gtk.PolicyType.AUTOMATIC)
-        scrolled.set_shadow_type(Gtk.ShadowType.IN)
-        self.dep_treeview = PackageDepView(self.depends_model, TYPE_DEP, "Build Depends")
+        self.dep_treeview = PackageDepView(self.depends_model, TYPE_DEP, "Dependencies")
         self.dep_treeview.connect("row-activated", self.on_package_activated, COL_DEP_PACKAGE)
         scrolled.add(self.dep_treeview)
         box.add(scrolled)
         pane.add2(box)
 
-        # Reverse Depends
+        # Reverse Task Depends
         scrolled = Gtk.ScrolledWindow()
         scrolled.set_policy(Gtk.PolicyType.AUTOMATIC, Gtk.PolicyType.AUTOMATIC)
         scrolled.set_shadow_type(Gtk.ShadowType.IN)
-        self.revdep_treeview = PackageReverseDepView(self.depends_model, "Reverse Depends")
+        self.revdep_treeview = PackageReverseDepView(self.depends_model, "Dependent Tasks")
         self.revdep_treeview.connect("row-activated", self.on_package_activated, COL_DEP_PARENT)
         scrolled.add(self.revdep_treeview)
         box.add(scrolled)
@@ -160,22 +151,15 @@
             current_package = None
         else:
             current_package = model.get_value(it, COL_PKG_NAME)
-        self.rdep_treeview.set_current_package(current_package)
         self.dep_treeview.set_current_package(current_package)
         self.revdep_treeview.set_current_package(current_package)
 
 
     def parse(self, depgraph):
-        for package in depgraph["pn"]:
-            self.pkg_model.insert(0, (package,))
-
-        for package in depgraph["depends"]:
-            for depend in depgraph["depends"][package]:
-                self.depends_model.insert (0, (TYPE_DEP, package, depend))
-
-        for package in depgraph["rdepends-pn"]:
-            for rdepend in depgraph["rdepends-pn"][package]:
-                self.depends_model.insert (0, (TYPE_RDEP, package, rdepend))
+        for task in depgraph["tdepends"]:
+            self.pkg_model.insert(0, (task,))
+            for depend in depgraph["tdepends"][task]:
+                self.depends_model.insert (0, (TYPE_DEP, task, depend))
 
 
 class gtkthread(threading.Thread):
@@ -313,7 +297,7 @@
                         extra = ". Close matches:\n  %s" % '\n  '.join(event._close_matches)
 
                 if event._dependees:
-                    print("Nothing %sPROVIDES '%s' (but %s %sDEPENDS on or otherwise requires it)%s" % r, event._item, ", ".join(event._dependees), r, extra)
+                    print("Nothing %sPROVIDES '%s' (but %s %sDEPENDS on or otherwise requires it)%s" % (r, event._item, ", ".join(event._dependees), r, extra))
                 else:
                     print("Nothing %sPROVIDES '%s'%s" % (r, event._item, extra))
                 if event._reasons:
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/toasterui.py b/import-layers/yocto-poky/bitbake/lib/bb/ui/toasterui.py
index 9808f6b..71f04fa 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/ui/toasterui.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/ui/toasterui.py
@@ -168,6 +168,9 @@
         logger.warning("buildhistory is not enabled. Please enable INHERIT += \"buildhistory\" to see image details.")
         build_history_enabled = False
 
+    if not "buildstats" in inheritlist.split(" "):
+        logger.warning("buildstats is not enabled. Please enable INHERIT += \"buildstats\" to generate build statistics.")
+
     if not params.observe_only:
         params.updateFromServer(server)
         params.updateToServer(server, os.environ.copy())
@@ -233,6 +236,9 @@
             # pylint: disable=protected-access
             # the code will look into the protected variables of the event; no easy way around this
 
+            if isinstance(event, bb.event.HeartbeatEvent):
+                continue
+
             if isinstance(event, bb.event.ParseStarted):
                 if not (build_log and build_log_file_path):
                     build_log, build_log_file_path = _open_build_log(log_dir)
@@ -432,9 +438,7 @@
                 elif event.type == "SetBRBE":
                     buildinfohelper.brbe = buildinfohelper._get_data_from_event(event)
                 elif event.type == "TaskArtifacts":
-                    # not implemented yet
-                    # see https://bugzilla.yoctoproject.org/show_bug.cgi?id=10283 for details
-                    pass
+                    buildinfohelper.scan_task_artifacts(event)
                 elif event.type == "OSErrorException":
                     logger.error(event)
                 else:
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/utils.py b/import-layers/yocto-poky/bitbake/lib/bb/utils.py
index 16fc9db..6a44db5 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/utils.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/utils.py
@@ -523,12 +523,8 @@
     """
     Return the hex string representation of the MD5 checksum of filename.
     """
-    try:
-        import hashlib
-        m = hashlib.md5()
-    except ImportError:
-        import md5
-        m = md5.new()
+    import hashlib
+    m = hashlib.md5()
 
     with open(filename, "rb") as f:
         for line in f:
@@ -538,14 +534,9 @@
 def sha256_file(filename):
     """
     Return the hex string representation of the 256-bit SHA checksum of
-    filename.  On Python 2.4 this will return None, so callers will need to
-    handle that by either skipping SHA checks, or running a standalone sha256sum
-    binary.
+    filename.
     """
-    try:
-        import hashlib
-    except ImportError:
-        return None
+    import hashlib
 
     s = hashlib.sha256()
     with open(filename, "rb") as f:
@@ -557,10 +548,7 @@
     """
     Return the hex string representation of the SHA1 checksum of the filename
     """
-    try:
-        import hashlib
-    except ImportError:
-        return None
+    import hashlib
 
     s = hashlib.sha1()
     with open(filename, "rb") as f:
@@ -665,7 +653,7 @@
     for var in bb.data.keys(d):
         export = d.getVarFlag(var, "export", False)
         if export:
-            os.environ[var] = d.getVar(var, True) or ""
+            os.environ[var] = d.getVar(var) or ""
 
 def _check_unsafe_delete_path(path):
     """
@@ -692,7 +680,7 @@
             if _check_unsafe_delete_path(path):
                 raise Exception('bb.utils.remove: called with dangerous path "%s" and recurse=True, refusing to delete!' % path)
         # shutil.rmtree(name) would be ideal but its too slow
-        subprocess.call(['rm', '-rf'] + glob.glob(path))
+        subprocess.check_call(['rm', '-rf'] + glob.glob(path))
         return
     for name in glob.glob(path):
         try:
@@ -911,11 +899,20 @@
         newmtime = sstat[stat.ST_MTIME]
     return newmtime
 
-def which(path, item, direction = 0, history = False):
+def which(path, item, direction = 0, history = False, executable=False):
     """
-    Locate a file in a PATH
+    Locate `item` in the list of paths `path` (colon separated string like $PATH).
+    If `direction` is non-zero then the list is reversed.
+    If `history` is True then the list of candidates also returned as result,history.
+    If `executable` is True then the candidate has to be an executable file,
+    otherwise the candidate simply has to exist.
     """
 
+    if executable:
+        is_candidate = lambda p: os.path.isfile(p) and os.access(p, os.X_OK)
+    else:
+        is_candidate = lambda p: os.path.exists(p)
+
     hist = []
     paths = (path or "").split(':')
     if direction != 0:
@@ -924,7 +921,7 @@
     for p in paths:
         next = os.path.join(p, item)
         hist.append(next)
-        if os.path.exists(next):
+        if is_candidate(next):
             if not os.path.isabs(next):
                 next = os.path.abspath(next)
             if history:
@@ -953,7 +950,7 @@
     Arguments:
 
     variable -- the variable name. This will be fetched and expanded (using
-    d.getVar(variable, True)) and then split into a set().
+    d.getVar(variable)) and then split into a set().
 
     checkvalues -- if this is a string it is split on whitespace into a set(),
     otherwise coerced directly into a set().
@@ -966,7 +963,7 @@
     d -- the data store.
     """
 
-    val = d.getVar(variable, True)
+    val = d.getVar(variable)
     if not val:
         return falsevalue
     val = set(val.split())
@@ -979,7 +976,7 @@
     return falsevalue
 
 def contains_any(variable, checkvalues, truevalue, falsevalue, d):
-    val = d.getVar(variable, True)
+    val = d.getVar(variable)
     if not val:
         return falsevalue
     val = set(val.split())
@@ -991,6 +988,30 @@
         return truevalue
     return falsevalue
 
+def filter(variable, checkvalues, d):
+    """Return all words in the variable that are present in the checkvalues.
+
+    Arguments:
+
+    variable -- the variable name. This will be fetched and expanded (using
+    d.getVar(variable)) and then split into a set().
+
+    checkvalues -- if this is a string it is split on whitespace into a set(),
+    otherwise coerced directly into a set().
+
+    d -- the data store.
+    """
+
+    val = d.getVar(variable)
+    if not val:
+        return ''
+    val = set(val.split())
+    if isinstance(checkvalues, str):
+        checkvalues = set(checkvalues.split())
+    else:
+        checkvalues = set(checkvalues)
+    return ' '.join(sorted(checkvalues & val))
+
 def cpu_count():
     return multiprocessing.cpu_count()
 
@@ -1378,10 +1399,10 @@
 
 def get_file_layer(filename, d):
     """Determine the collection (as defined by a layer's layer.conf file) containing the specified file"""
-    collections = (d.getVar('BBFILE_COLLECTIONS', True) or '').split()
+    collections = (d.getVar('BBFILE_COLLECTIONS') or '').split()
     collection_res = {}
     for collection in collections:
-        collection_res[collection] = d.getVar('BBFILE_PATTERN_%s' % collection, True) or ''
+        collection_res[collection] = d.getVar('BBFILE_PATTERN_%s' % collection) or ''
 
     def path_to_layer(path):
         # Use longest path so we handle nested layers
@@ -1394,7 +1415,7 @@
         return match
 
     result = None
-    bbfiles = (d.getVar('BBFILES', True) or '').split()
+    bbfiles = (d.getVar('BBFILES') or '').split()
     bbfilesmatch = False
     for bbfilesentry in bbfiles:
         if fnmatch.fnmatch(filename, bbfilesentry):
@@ -1471,7 +1492,7 @@
         if v in os.environ.keys():
             exported = True
         else:
-            v_proxy = d.getVar(v, True)
+            v_proxy = d.getVar(v)
             if v_proxy is not None:
                 os.environ[v] = v_proxy
                 exported = True
@@ -1503,3 +1524,14 @@
                 plugins.append(obj or plugin)
             else:
                 plugins.append(plugin)
+
+
+class LogCatcher(logging.Handler):
+    """Logging handler for collecting logged messages so you can check them later"""
+    def __init__(self):
+        self.messages = []
+        logging.Handler.__init__(self, logging.WARNING)
+    def emit(self, record):
+        self.messages.append(bb.build.logformatter.format(record))
+    def contains(self, message):
+        return (message in self.messages)
diff --git a/import-layers/yocto-poky/bitbake/lib/bblayers/action.py b/import-layers/yocto-poky/bitbake/lib/bblayers/action.py
index 739ae27..cf94704 100644
--- a/import-layers/yocto-poky/bitbake/lib/bblayers/action.py
+++ b/import-layers/yocto-poky/bitbake/lib/bblayers/action.py
@@ -180,7 +180,7 @@
 
         if first_regex:
             # Find the BBFILES entries that match (which will have come from this conf/layer.conf file)
-            bbfiles = str(self.tinfoil.config_data.getVar('BBFILES', True)).split()
+            bbfiles = str(self.tinfoil.config_data.getVar('BBFILES')).split()
             bbfiles_layer = []
             for item in bbfiles:
                 if first_regex.match(item):
diff --git a/import-layers/yocto-poky/bitbake/lib/bblayers/common.py b/import-layers/yocto-poky/bitbake/lib/bblayers/common.py
index b10fb4c..98515ce 100644
--- a/import-layers/yocto-poky/bitbake/lib/bblayers/common.py
+++ b/import-layers/yocto-poky/bitbake/lib/bblayers/common.py
@@ -12,7 +12,7 @@
 
     def tinfoil_init(self, tinfoil):
         self.tinfoil = tinfoil
-        self.bblayers = (self.tinfoil.config_data.getVar('BBLAYERS', True) or "").split()
+        self.bblayers = (self.tinfoil.config_data.getVar('BBLAYERS') or "").split()
         layerconfs = self.tinfoil.config_data.varhistory.get_variable_items_files('BBFILE_COLLECTIONS', self.tinfoil.config_data)
         self.bbfile_collections = {layer: os.path.dirname(os.path.dirname(path)) for layer, path in layerconfs.items()}
 
diff --git a/import-layers/yocto-poky/bitbake/lib/bblayers/layerindex.py b/import-layers/yocto-poky/bitbake/lib/bblayers/layerindex.py
index 10ad718..506c110 100644
--- a/import-layers/yocto-poky/bitbake/lib/bblayers/layerindex.py
+++ b/import-layers/yocto-poky/bitbake/lib/bblayers/layerindex.py
@@ -56,7 +56,7 @@
         r = conn.getresponse()
         if r.status != 200:
             raise Exception("Failed to read " + path + ": %d %s" % (r.status, r.reason))
-        return json.loads(r.read())
+        return json.loads(r.read().decode())
 
     def get_layer_deps(self, layername, layeritems, layerbranches, layerdependencies, branchnum, selfname=False):
         def layeritems_info_id(items_name, layeritems):
@@ -151,7 +151,7 @@
     def do_layerindex_fetch(self, args):
         """Fetches a layer from a layer index along with its dependent layers, and adds them to conf/bblayers.conf.
 """
-        apiurl = self.tinfoil.config_data.getVar('BBLAYERS_LAYERINDEX_URL', True)
+        apiurl = self.tinfoil.config_data.getVar('BBLAYERS_LAYERINDEX_URL')
         if not apiurl:
             logger.error("Cannot get BBLAYERS_LAYERINDEX_URL")
             return 1
@@ -173,8 +173,8 @@
             return 1
 
         ignore_layers = []
-        for collection in self.tinfoil.config_data.getVar('BBFILE_COLLECTIONS', True).split():
-            lname = self.tinfoil.config_data.getVar('BBLAYERS_LAYERINDEX_NAME_%s' % collection, True)
+        for collection in self.tinfoil.config_data.getVar('BBFILE_COLLECTIONS').split():
+            lname = self.tinfoil.config_data.getVar('BBLAYERS_LAYERINDEX_NAME_%s' % collection)
             if lname:
                 ignore_layers.append(lname)
 
@@ -225,7 +225,7 @@
             printedlayers.append(dependency)
 
         if repourls:
-            fetchdir = self.tinfoil.config_data.getVar('BBLAYERS_FETCH_DIR', True)
+            fetchdir = self.tinfoil.config_data.getVar('BBLAYERS_FETCH_DIR')
             if not fetchdir:
                 logger.error("Cannot get BBLAYERS_FETCH_DIR")
                 return 1
diff --git a/import-layers/yocto-poky/bitbake/lib/bblayers/query.py b/import-layers/yocto-poky/bitbake/lib/bblayers/query.py
index ee1e7c8..bef3af3 100644
--- a/import-layers/yocto-poky/bitbake/lib/bblayers/query.py
+++ b/import-layers/yocto-poky/bitbake/lib/bblayers/query.py
@@ -5,8 +5,6 @@
 import os
 import re
 
-import bb.cache
-import bb.providers
 import bb.utils
 
 from bblayers.common import LayerPlugin
@@ -62,7 +60,7 @@
         # factor - however, each layer.conf is free to either prepend or append to
         # BBPATH (or indeed do crazy stuff with it). Thus the order in BBPATH might
         # not be exactly the order present in bblayers.conf either.
-        bbpath = str(self.tinfoil.config_data.getVar('BBPATH', True))
+        bbpath = str(self.tinfoil.config_data.getVar('BBPATH'))
         overlayed_class_found = False
         for (classfile, classdirs) in classes.items():
             if len(classdirs) > 1:
@@ -114,7 +112,7 @@
 
     def list_recipes(self, title, pnspec, show_overlayed_only, show_same_ver_only, show_filenames, show_multi_provider_only, inherits):
         if inherits:
-            bbpath = str(self.tinfoil.config_data.getVar('BBPATH', True))
+            bbpath = str(self.tinfoil.config_data.getVar('BBPATH'))
             for classname in inherits:
                 classfile = 'classes/%s.bbclass' % classname
                 if not bb.utils.which(bbpath, classfile, history=False):
@@ -122,15 +120,13 @@
                     sys.exit(1)
 
         pkg_pn = self.tinfoil.cooker.recipecaches[''].pkg_pn
-        (latest_versions, preferred_versions) = bb.providers.findProviders(self.tinfoil.config_data, self.tinfoil.cooker.recipecaches[''], pkg_pn)
-        allproviders = bb.providers.allProviders(self.tinfoil.cooker.recipecaches[''])
+        (latest_versions, preferred_versions) = self.tinfoil.find_providers()
+        allproviders = self.tinfoil.get_all_providers()
 
         # Ensure we list skipped recipes
         # We are largely guessing about PN, PV and the preferred version here,
         # but we have no choice since skipped recipes are not fully parsed
         skiplist = list(self.tinfoil.cooker.skiplist.keys())
-        skiplist.sort( key=lambda fileitem: self.tinfoil.cooker.collection.calc_bbfile_priority(fileitem) )
-        skiplist.reverse()
         for fn in skiplist:
             recipe_parts = os.path.splitext(os.path.basename(fn))[0].split('_')
             p = recipe_parts[0]
@@ -158,7 +154,7 @@
                     logger.plain("%s:", pn)
                 logger.plain("  %s %s%s", layer.ljust(20), ver, skipped)
 
-        global_inherit = (self.tinfoil.config_data.getVar('INHERIT', True) or "").split()
+        global_inherit = (self.tinfoil.config_data.getVar('INHERIT') or "").split()
         cls_re = re.compile('classes/')
 
         preffiles = []
@@ -246,17 +242,22 @@
 
 Lists recipes with the bbappends that apply to them as subitems.
 """
-
-        logger.plain('=== Appended recipes ===')
+        if args.pnspec:
+            logger.plain('=== Matched appended recipes ===')
+        else:
+            logger.plain('=== Appended recipes ===')
 
         pnlist = list(self.tinfoil.cooker_data.pkg_pn.keys())
         pnlist.sort()
         appends = False
         for pn in pnlist:
+            if args.pnspec and pn != args.pnspec:
+                continue
+
             if self.show_appends_for_pn(pn):
                 appends = True
 
-        if self.show_appends_for_skipped():
+        if not args.pnspec and self.show_appends_for_skipped():
             appends = True
 
         if not appends:
@@ -265,10 +266,7 @@
     def show_appends_for_pn(self, pn):
         filenames = self.tinfoil.cooker_data.pkg_pn[pn]
 
-        best = bb.providers.findBestProvider(pn,
-                                             self.tinfoil.config_data,
-                                             self.tinfoil.cooker_data,
-                                             self.tinfoil.cooker_data.pkg_pn)
+        best = self.tinfoil.find_best_provider(pn)
         best_filename = os.path.basename(best[3])
 
         return self.show_appends_output(filenames, best_filename)
@@ -319,12 +317,12 @@
         ignore_layers = (args.ignore or '').split(',')
 
         pkg_fn = self.tinfoil.cooker_data.pkg_fn
-        bbpath = str(self.tinfoil.config_data.getVar('BBPATH', True))
+        bbpath = str(self.tinfoil.config_data.getVar('BBPATH'))
         self.require_re = re.compile(r"require\s+(.+)")
         self.include_re = re.compile(r"include\s+(.+)")
         self.inherit_re = re.compile(r"inherit\s+(.+)")
 
-        global_inherit = (self.tinfoil.config_data.getVar('INHERIT', True) or "").split()
+        global_inherit = (self.tinfoil.config_data.getVar('INHERIT') or "").split()
 
         # The bb's DEPENDS and RDEPENDS
         for f in pkg_fn:
@@ -336,10 +334,7 @@
             deps = self.tinfoil.cooker_data.deps[f]
             for pn in deps:
                 if pn in self.tinfoil.cooker_data.pkg_pn:
-                    best = bb.providers.findBestProvider(pn,
-                            self.tinfoil.config_data,
-                            self.tinfoil.cooker_data,
-                            self.tinfoil.cooker_data.pkg_pn)
+                    best = self.tinfoil.find_best_provider(pn)
                     self.check_cross_depends("DEPENDS", layername, f, best[3], args.filenames, ignore_layers)
 
             # The RDPENDS
@@ -352,14 +347,11 @@
                     sorted_rdeps[k2] = 1
             all_rdeps = sorted_rdeps.keys()
             for rdep in all_rdeps:
-                all_p = bb.providers.getRuntimeProviders(self.tinfoil.cooker_data, rdep)
+                all_p, best = self.tinfoil.get_runtime_providers(rdep)
                 if all_p:
                     if f in all_p:
                         # The recipe provides this one itself, ignore
                         continue
-                    best = bb.providers.filterProvidersRunTime(all_p, rdep,
-                                    self.tinfoil.config_data,
-                                    self.tinfoil.cooker_data)[0][0]
                     self.check_cross_depends("RDEPENDS", layername, f, best, args.filenames, ignore_layers)
 
             # The RRECOMMENDS
@@ -372,14 +364,11 @@
                     sorted_rrecs[k2] = 1
             all_rrecs = sorted_rrecs.keys()
             for rrec in all_rrecs:
-                all_p = bb.providers.getRuntimeProviders(self.tinfoil.cooker_data, rrec)
+                all_p, best = self.tinfoil.get_runtime_providers(rrec)
                 if all_p:
                     if f in all_p:
                         # The recipe provides this one itself, ignore
                         continue
-                    best = bb.providers.filterProvidersRunTime(all_p, rrec,
-                                    self.tinfoil.config_data,
-                                    self.tinfoil.cooker_data)[0][0]
                     self.check_cross_depends("RRECOMMENDS", layername, f, best, args.filenames, ignore_layers)
 
             # The inherit class
@@ -493,7 +482,8 @@
         parser_show_recipes.add_argument('-i', '--inherits', help='only list recipes that inherit the named class', metavar='CLASS', default='')
         parser_show_recipes.add_argument('pnspec', nargs='?', help='optional recipe name specification (wildcards allowed, enclose in quotes to avoid shell expansion)')
 
-        self.add_command(sp, 'show-appends', self.do_show_appends)
+        parser_show_appends = self.add_command(sp, 'show-appends', self.do_show_appends)
+        parser_show_appends.add_argument('pnspec', nargs='?', help='optional recipe name specification (wildcards allowed, enclose in quotes to avoid shell expansion)')
 
         parser_show_cross_depends = self.add_command(sp, 'show-cross-depends', self.do_show_cross_depends)
         parser_show_cross_depends.add_argument('-f', '--filenames', help='show full file path', action='store_true')
diff --git a/import-layers/yocto-poky/bitbake/lib/prserv/serv.py b/import-layers/yocto-poky/bitbake/lib/prserv/serv.py
index cafcc82..a7efa58 100644
--- a/import-layers/yocto-poky/bitbake/lib/prserv/serv.py
+++ b/import-layers/yocto-poky/bitbake/lib/prserv/serv.py
@@ -5,12 +5,7 @@
 import queue
 import socket
 import io
-
-try:
-    import sqlite3
-except ImportError:
-    from pysqlite2 import dbapi2 as sqlite3
-
+import sqlite3
 import bb.server.xmlrpc
 import prserv
 import prserv.db
@@ -242,12 +237,25 @@
 
         sys.stdout.flush()
         sys.stderr.flush()
+
+        # We could be called from a python thread with io.StringIO as
+        # stdout/stderr or it could be 'real' unix fd forking where we need
+        # to physically close the fds to prevent the program launching us from
+        # potentially hanging on a pipe. Handle both cases.
         si = open('/dev/null', 'r')
+        try:
+            os.dup2(si.fileno(),sys.stdin.fileno())
+        except (AttributeError, io.UnsupportedOperation):
+            sys.stdin = si
         so = open(self.logfile, 'a+')
-        se = so
-        os.dup2(si.fileno(),sys.stdin.fileno())
-        os.dup2(so.fileno(),sys.stdout.fileno())
-        os.dup2(se.fileno(),sys.stderr.fileno())
+        try:
+            os.dup2(so.fileno(),sys.stdout.fileno())
+        except (AttributeError, io.UnsupportedOperation):
+            sys.stdout = so
+        try:
+            os.dup2(so.fileno(),sys.stderr.fileno())
+        except (AttributeError, io.UnsupportedOperation):
+            sys.stderr = so
 
         # Clear out all log handlers prior to the fork() to avoid calling
         # event handlers not part of the PRserver
@@ -420,7 +428,7 @@
 def auto_start(d):
     global singleton
 
-    host_params = list(filter(None, (d.getVar('PRSERV_HOST', True) or '').split(':')))
+    host_params = list(filter(None, (d.getVar('PRSERV_HOST') or '').split(':')))
     if not host_params:
         return None
 
@@ -431,7 +439,7 @@
 
     if is_local_special(host_params[0], int(host_params[1])) and not singleton:
         import bb.utils
-        cachedir = (d.getVar("PERSISTENT_DIR", True) or d.getVar("CACHE", True))
+        cachedir = (d.getVar("PERSISTENT_DIR") or d.getVar("CACHE"))
         if not cachedir:
             logger.critical("Please set the 'PERSISTENT_DIR' or 'CACHE' variable")
             raise PRServiceConfigError
diff --git a/import-layers/yocto-poky/bitbake/lib/simplediff/LICENSE b/import-layers/yocto-poky/bitbake/lib/simplediff/LICENSE
new file mode 100644
index 0000000..8242dde
--- /dev/null
+++ b/import-layers/yocto-poky/bitbake/lib/simplediff/LICENSE
@@ -0,0 +1,22 @@
+Copyright (c) 2008 - 2013 Paul Butler and contributors
+
+This sofware may be used under a zlib/libpng-style license:
+
+This software is provided 'as-is', without any express or implied warranty. In
+no event will the authors be held liable for any damages arising from the use
+of this software.
+
+Permission is granted to anyone to use this software for any purpose, including
+commercial applications, and to alter it and redistribute it freely, subject to
+the following restrictions:
+
+1. The origin of this software must not be misrepresented; you must not claim
+that you wrote the original software. If you use this software in a product, an
+acknowledgment in the product documentation would be appreciated but is not
+required.
+
+2. Altered source versions must be plainly marked as such, and must not be
+misrepresented as being the original software.
+
+3. This notice may not be removed or altered from any source distribution.
+
diff --git a/import-layers/yocto-poky/bitbake/lib/simplediff/__init__.py b/import-layers/yocto-poky/bitbake/lib/simplediff/__init__.py
new file mode 100644
index 0000000..57ee3c5
--- /dev/null
+++ b/import-layers/yocto-poky/bitbake/lib/simplediff/__init__.py
@@ -0,0 +1,198 @@
+'''
+Simple Diff for Python version 1.0
+
+Annotate two versions of a list with the values that have been
+changed between the versions, similar to unix's `diff` but with
+a dead-simple Python interface.
+
+(C) Paul Butler 2008-2012 <http://www.paulbutler.org/>
+May be used and distributed under the zlib/libpng license
+<http://www.opensource.org/licenses/zlib-license.php>
+'''
+
+__all__ = ['diff', 'string_diff', 'html_diff']
+__version__ = '1.0'
+
+
+def diff(old, new):
+    '''
+    Find the differences between two lists. Returns a list of pairs, where the
+    first value is in ['+','-','='] and represents an insertion, deletion, or
+    no change for that list. The second value of the pair is the list
+    of elements.
+
+    Params:
+        old     the old list of immutable, comparable values (ie. a list
+                of strings)
+        new     the new list of immutable, comparable values
+   
+    Returns:
+        A list of pairs, with the first part of the pair being one of three
+        strings ('-', '+', '=') and the second part being a list of values from
+        the original old and/or new lists. The first part of the pair
+        corresponds to whether the list of values is a deletion, insertion, or
+        unchanged, respectively.
+
+    Examples:
+        >>> diff([1,2,3,4],[1,3,4])
+        [('=', [1]), ('-', [2]), ('=', [3, 4])]
+
+        >>> diff([1,2,3,4],[2,3,4,1])
+        [('-', [1]), ('=', [2, 3, 4]), ('+', [1])]
+
+        >>> diff('The quick brown fox jumps over the lazy dog'.split(),
+        ...      'The slow blue cheese drips over the lazy carrot'.split())
+        ... # doctest: +NORMALIZE_WHITESPACE
+        [('=', ['The']),
+         ('-', ['quick', 'brown', 'fox', 'jumps']),
+         ('+', ['slow', 'blue', 'cheese', 'drips']),
+         ('=', ['over', 'the', 'lazy']),
+         ('-', ['dog']),
+         ('+', ['carrot'])]
+
+    '''
+
+    # Create a map from old values to their indices
+    old_index_map = dict()
+    for i, val in enumerate(old):
+        old_index_map.setdefault(val,list()).append(i)
+
+    # Find the largest substring common to old and new.
+    # We use a dynamic programming approach here.
+    # 
+    # We iterate over each value in the `new` list, calling the
+    # index `inew`. At each iteration, `overlap[i]` is the
+    # length of the largest suffix of `old[:i]` equal to a suffix
+    # of `new[:inew]` (or unset when `old[i]` != `new[inew]`).
+    #
+    # At each stage of iteration, the new `overlap` (called
+    # `_overlap` until the original `overlap` is no longer needed)
+    # is built from the old one.
+    #
+    # If the length of overlap exceeds the largest substring
+    # seen so far (`sub_length`), we update the largest substring
+    # to the overlapping strings.
+
+    overlap = dict()
+    # `sub_start_old` is the index of the beginning of the largest overlapping
+    # substring in the old list. `sub_start_new` is the index of the beginning
+    # of the same substring in the new list. `sub_length` is the length that
+    # overlaps in both.
+    # These track the largest overlapping substring seen so far, so naturally
+    # we start with a 0-length substring.
+    sub_start_old = 0
+    sub_start_new = 0
+    sub_length = 0
+
+    for inew, val in enumerate(new):
+        _overlap = dict()
+        for iold in old_index_map.get(val,list()):
+            # now we are considering all values of iold such that
+            # `old[iold] == new[inew]`.
+            _overlap[iold] = (iold and overlap.get(iold - 1, 0)) + 1
+            if(_overlap[iold] > sub_length):
+                # this is the largest substring seen so far, so store its
+                # indices
+                sub_length = _overlap[iold]
+                sub_start_old = iold - sub_length + 1
+                sub_start_new = inew - sub_length + 1
+        overlap = _overlap
+
+    if sub_length == 0:
+        # If no common substring is found, we return an insert and delete...
+        return (old and [('-', old)] or []) + (new and [('+', new)] or [])
+    else:
+        # ...otherwise, the common substring is unchanged and we recursively
+        # diff the text before and after that substring
+        return diff(old[ : sub_start_old], new[ : sub_start_new]) + \
+               [('=', new[sub_start_new : sub_start_new + sub_length])] + \
+               diff(old[sub_start_old + sub_length : ],
+                       new[sub_start_new + sub_length : ])
+
+
+def string_diff(old, new):
+    '''
+    Returns the difference between the old and new strings when split on
+    whitespace. Considers punctuation a part of the word
+
+    This function is intended as an example; you'll probably want
+    a more sophisticated wrapper in practice.
+
+    Params:
+        old     the old string
+        new     the new string
+
+    Returns:
+        the output of `diff` on the two strings after splitting them
+        on whitespace (a list of change instructions; see the docstring
+        of `diff`)
+
+    Examples:
+        >>> string_diff('The quick brown fox', 'The fast blue fox')
+        ... # doctest: +NORMALIZE_WHITESPACE
+        [('=', ['The']),
+         ('-', ['quick', 'brown']),
+         ('+', ['fast', 'blue']),
+         ('=', ['fox'])]
+
+    '''
+    return diff(old.split(), new.split())
+
+
+def html_diff(old, new):
+    '''
+    Returns the difference between two strings (as in stringDiff) in
+    HTML format. HTML code in the strings is NOT escaped, so you
+    will get weird results if the strings contain HTML.
+
+    This function is intended as an example; you'll probably want
+    a more sophisticated wrapper in practice.
+
+    Params:
+        old     the old string
+        new     the new string
+
+    Returns:
+        the output of the diff expressed with HTML <ins> and <del>
+        tags.
+
+    Examples:
+        >>> html_diff('The quick brown fox', 'The fast blue fox')
+        'The <del>quick brown</del> <ins>fast blue</ins> fox'
+    '''
+    con = {'=': (lambda x: x),
+           '+': (lambda x: "<ins>" + x + "</ins>"),
+           '-': (lambda x: "<del>" + x + "</del>")}
+    return " ".join([(con[a])(" ".join(b)) for a, b in string_diff(old, new)])
+
+
+def check_diff(old, new):
+    '''
+    This tests that diffs returned by `diff` are valid. You probably won't
+    want to use this function, but it's provided for documentation and
+    testing.
+
+    A diff should satisfy the property that the old input is equal to the
+    elements of the result annotated with '-' or '=' concatenated together.
+    Likewise, the new input is equal to the elements of the result annotated
+    with '+' or '=' concatenated together. This function compares `old`,
+    `new`, and the results of `diff(old, new)` to ensure this is true.
+
+    Tests:
+        >>> check_diff('ABCBA', 'CBABA')
+        >>> check_diff('Foobarbaz', 'Foobarbaz')
+        >>> check_diff('Foobarbaz', 'Boobazbam')
+        >>> check_diff('The quick brown fox', 'Some quick brown car')
+        >>> check_diff('A thick red book', 'A quick blue book')
+        >>> check_diff('dafhjkdashfkhasfjsdafdasfsda', 'asdfaskjfhksahkfjsdha')
+        >>> check_diff('88288822828828288282828', '88288882882828282882828')
+        >>> check_diff('1234567890', '24689')
+    '''
+    old = list(old)
+    new = list(new)
+    result = diff(old, new)
+    _old = [val for (a, vals) in result if (a in '=-') for val in vals]
+    assert old == _old, 'Expected %s, got %s' % (old, _old)
+    _new = [val for (a, vals) in result if (a in '=+') for val in vals]
+    assert new == _new, 'Expected %s, got %s' % (new, _new)
+
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
index e5f7c98..1207102 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
@@ -27,7 +27,7 @@
 from django.db import transaction
 from django.db.models import Q
 from bldcontrol.models import BuildEnvironment, BRLayer, BRVariable, BRTarget, BRBitbake
-from orm.models import CustomImageRecipe, Layer, Layer_Version, ProjectLayer
+from orm.models import CustomImageRecipe, Layer, Layer_Version, ProjectLayer, ToasterSetting
 import subprocess
 
 from toastermain import settings
@@ -200,72 +200,93 @@
 
         logger.debug("localhostbecontroller: current layer list %s " % pformat(layerlist))
 
+        if self.pokydirname is None and os.path.exists(os.path.join(self.be.sourcedir, "oe-init-build-env")):
+            logger.debug("localhostbecontroller: selected poky dir name %s" % self.be.sourcedir)
+            self.pokydirname = self.be.sourcedir
+
         # 5. create custom layer and add custom recipes to it
-        layerpath = os.path.join(self.be.builddir,
-                                 CustomImageRecipe.LAYER_NAME)
         for target in targets:
             try:
-                customrecipe = CustomImageRecipe.objects.get(name=target.target,
-                                                             project=bitbake.req.project)
+                customrecipe = CustomImageRecipe.objects.get(
+                    name=target.target,
+                    project=bitbake.req.project)
+
+                custom_layer_path = self.setup_custom_image_recipe(
+                    customrecipe, layers)
+
+                if os.path.isdir(custom_layer_path):
+                    layerlist.append(custom_layer_path)
+
             except CustomImageRecipe.DoesNotExist:
-                continue # not a custom recipe, skip
+                continue  # not a custom recipe, skip
 
-            # create directory structure
-            for name in ("conf", "recipes"):
-                path = os.path.join(layerpath, name)
-                if not os.path.isdir(path):
-                    os.makedirs(path)
-
-            # create layer.oonf
-            config = os.path.join(layerpath, "conf", "layer.conf")
-            if not os.path.isfile(config):
-                with open(config, "w") as conf:
-                    conf.write('BBPATH .= ":${LAYERDIR}"\nBBFILES += "${LAYERDIR}/recipes/*.bb"\n')
-
-            # Update the Layer_Version dirpath that has our base_recipe in
-            # to be able to read the base recipe to then  generate the
-            # custom recipe.
-            br_layer_base_recipe = layers.get(
-                layer_version=customrecipe.base_recipe.layer_version)
-
-            br_layer_base_dirpath = \
-                    os.path.join(self.be.sourcedir,
-                                 self.getGitCloneDirectory(
-                                     br_layer_base_recipe.giturl,
-                                     br_layer_base_recipe.commit),
-                                 customrecipe.base_recipe.layer_version.dirpath
-                                )
-
-            customrecipe.base_recipe.layer_version.dirpath = \
-                         br_layer_base_dirpath
-
-            customrecipe.base_recipe.layer_version.save()
-
-            # create recipe
-            recipe_path = \
-                    os.path.join(layerpath, "recipes", "%s.bb" % target.target)
-            with open(recipe_path, "w") as recipef:
-                recipef.write(customrecipe.generate_recipe_file_contents())
-
-            # Update the layer and recipe objects
-            customrecipe.layer_version.dirpath = layerpath
-            customrecipe.layer_version.save()
-
-            customrecipe.file_path = recipe_path
-            customrecipe.save()
-
-            # create *Layer* objects needed for build machinery to work
-            BRLayer.objects.get_or_create(req=target.req,
-                                          name=layer.name,
-                                          dirpath=layerpath,
-                                          giturl="file://%s" % layerpath)
-        if os.path.isdir(layerpath):
-            layerlist.append(layerpath)
-
-        self.islayerset = True
         layerlist.extend(nongitlayerlist)
+        logger.debug("\n\nset layers gives this list %s" % pformat(layerlist))
+        self.islayerset = True
         return layerlist
 
+    def setup_custom_image_recipe(self, customrecipe, layers):
+        """ Set up toaster-custom-images layer and recipe files """
+        layerpath = os.path.join(self.be.builddir,
+                                 CustomImageRecipe.LAYER_NAME)
+
+        # create directory structure
+        for name in ("conf", "recipes"):
+            path = os.path.join(layerpath, name)
+            if not os.path.isdir(path):
+                os.makedirs(path)
+
+        # create layer.conf
+        config = os.path.join(layerpath, "conf", "layer.conf")
+        if not os.path.isfile(config):
+            with open(config, "w") as conf:
+                conf.write('BBPATH .= ":${LAYERDIR}"\nBBFILES += "${LAYERDIR}/recipes/*.bb"\n')
+
+        # Update the Layer_Version dirpath that has our base_recipe in
+        # to be able to read the base recipe to then  generate the
+        # custom recipe.
+        br_layer_base_recipe = layers.get(
+            layer_version=customrecipe.base_recipe.layer_version)
+
+        # If the layer is one that we've cloned we know where it lives
+        if br_layer_base_recipe.giturl and br_layer_base_recipe.commit:
+            layer_path = self.getGitCloneDirectory(
+                br_layer_base_recipe.giturl,
+                br_layer_base_recipe.commit)
+        # Otherwise it's a local layer
+        elif br_layer_base_recipe.local_source_dir:
+            layer_path = br_layer_base_recipe.local_source_dir
+        else:
+            logger.error("Unable to workout the dir path for the custom"
+                         " image recipe")
+
+        br_layer_base_dirpath = os.path.join(
+            self.be.sourcedir,
+            layer_path,
+            customrecipe.base_recipe.layer_version.dirpath)
+
+        customrecipe.base_recipe.layer_version.dirpath = br_layer_base_dirpath
+
+        customrecipe.base_recipe.layer_version.save()
+
+        # create recipe
+        recipe_path = os.path.join(layerpath, "recipes", "%s.bb" %
+                                   customrecipe.name)
+        with open(recipe_path, "w") as recipef:
+            recipef.write(customrecipe.generate_recipe_file_contents())
+
+        # Update the layer and recipe objects
+        customrecipe.layer_version.dirpath = layerpath
+        customrecipe.layer_version.layer.local_source_dir = layerpath
+        customrecipe.layer_version.layer.save()
+        customrecipe.layer_version.save()
+
+        customrecipe.file_path = recipe_path
+        customrecipe.save()
+
+        return layerpath
+
+
     def readServerLogFile(self):
         return open(os.path.join(self.be.builddir, "toaster_server.log"), "r").read()
 
@@ -277,23 +298,17 @@
         builddir = '%s-toaster-%d' % (self.be.builddir, bitbake.req.project.id)
         oe_init = os.path.join(self.pokydirname, 'oe-init-build-env')
         # init build environment
-        self._shellcmd("bash -c 'source %s %s'" % (oe_init, builddir),
+        try:
+            custom_script = ToasterSetting.objects.get(name="CUSTOM_BUILD_INIT_SCRIPT").value
+            custom_script = custom_script.replace("%BUILDDIR%" ,builddir)
+            self._shellcmd("bash -c 'source %s'" % (custom_script))
+        except ToasterSetting.DoesNotExist:
+            self._shellcmd("bash -c 'source %s %s'" % (oe_init, builddir),
                        self.be.sourcedir)
 
         # update bblayers.conf
-        bblconfpath = os.path.join(builddir, "conf/bblayers.conf")
-        conflines = open(bblconfpath, "r").readlines()
-        skip = False
+        bblconfpath = os.path.join(builddir, "conf/toaster-bblayers.conf")
         with open(bblconfpath, 'w') as bblayers:
-            for line in conflines:
-                if line.startswith("# line added by toaster"):
-                    skip = True
-                    continue
-                if skip:
-                    skip = False
-                else:
-                    bblayers.write(line)
-
             bblayers.write('# line added by toaster build control\n'
                            'BBLAYERS = "%s"' % ' '.join(layers))
 
@@ -306,9 +321,10 @@
 
         # run bitbake server from the clone
         bitbake = os.path.join(self.pokydirname, 'bitbake', 'bin', 'bitbake')
-        self._shellcmd('bash -c \"source %s %s; BITBAKE_UI="knotty" %s --read %s '
+        toasterlayers = os.path.join(builddir,"conf/toaster-bblayers.conf")
+        self._shellcmd('bash -c \"source %s %s; BITBAKE_UI="knotty" %s --read %s --read %s '
                        '--server-only -t xmlrpc -B 0.0.0.0:0\"' % (oe_init,
-                       builddir, bitbake, confpath), self.be.sourcedir)
+                       builddir, bitbake, confpath, toasterlayers), self.be.sourcedir)
 
         # read port number from bitbake.lock
         self.be.bbport = ""
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py
index 7f7a5a9..df11f9d 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py
@@ -11,9 +11,11 @@
 import logging
 import traceback
 import signal
+import os
 
 logger = logging.getLogger("toaster")
 
+
 class Command(NoArgsCommand):
     args = ""
     help = "Schedules and executes build requests as possible. "\
@@ -50,7 +52,7 @@
                 logger.debug("runbuilds: No build env")
                 return
 
-            logger.info("runbuilds: starting build %s, environment %s" % \
+            logger.info("runbuilds: starting build %s, environment %s" %
                         (br, bec.be))
 
             # let the build request know where it is being executed
@@ -80,7 +82,7 @@
 
     def archive(self):
         for br in BuildRequest.objects.filter(state=BuildRequest.REQ_ARCHIVE):
-            if br.build == None:
+            if br.build is None:
                 br.state = BuildRequest.REQ_FAILED
             else:
                 br.state = BuildRequest.REQ_COMPLETED
@@ -99,10 +101,10 @@
             Q(updated__lt=timezone.now() - timedelta(seconds=30))
         ).update(lock=BuildEnvironment.LOCK_FREE)
 
-
         # update all Builds that were in progress and failed to start
-        for br in BuildRequest.objects.filter(state=BuildRequest.REQ_FAILED,
-                                              build__outcome=Build.IN_PROGRESS):
+        for br in BuildRequest.objects.filter(
+                state=BuildRequest.REQ_FAILED,
+                build__outcome=Build.IN_PROGRESS):
             # transpose the launch errors in ToasterExceptions
             br.build.outcome = Build.FAILED
             for brerror in br.brerror_set.all():
@@ -117,7 +119,6 @@
             br.environment.lock = BuildEnvironment.LOCK_FREE
             br.environment.save()
 
-
         # update all BuildRequests without a build created
         for br in BuildRequest.objects.filter(build=None):
             br.build = Build.objects.create(project=br.project,
@@ -144,7 +145,7 @@
 
         # Make sure the LOCK is removed for builds which have been fully
         # cancelled
-        for br in BuildRequest.objects.filter(\
+        for br in BuildRequest.objects.filter(
                       Q(build__outcome=Build.CANCELLED) &
                       Q(state=BuildRequest.REQ_CANCELLING) &
                       ~Q(environment=None)):
@@ -168,6 +169,12 @@
             logger.warn("runbuilds: schedule exception %s" % str(e))
 
     def handle_noargs(self, **options):
+        pidfile_path = os.path.join(os.environ.get("BUILDDIR", "."),
+                                    ".runbuilds.pid")
+
+        with open(pidfile_path, 'w') as pidfile:
+            pidfile.write("%s" % os.getpid())
+
         self.runbuild()
 
         signal.signal(signal.SIGUSR1, lambda sig, frame: None)
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/tests.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/tests.py
deleted file mode 100644
index 475ac0a..0000000
--- a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/tests.py
+++ /dev/null
@@ -1,141 +0,0 @@
-"""
-This file demonstrates writing tests using the unittest module. These will pass
-when you run "manage.py test".
-
-Replace this with more appropriate tests for your application.
-"""
-
-from django.test import TestCase
-
-from bldcontrol.bbcontroller import BitbakeController, BuildSetupException
-from bldcontrol.localhostbecontroller import LocalhostBEController
-from bldcontrol.models import BuildEnvironment, BuildRequest
-from bldcontrol.management.commands.runbuilds import Command
-
-import socket
-import subprocess
-import os
-
-# standard poky data hardcoded for testing
-BITBAKE_LAYER = type('bitbake_info', (object,), { "giturl": "git://git.yoctoproject.org/poky.git", "dirpath": "", "commit": "HEAD"})
-POKY_LAYERS = [
-    type('poky_info', (object,), { "name": "meta", "giturl": "git://git.yoctoproject.org/poky.git", "dirpath": "meta", "commit": "HEAD"}),
-    type('poky_info', (object,), { "name": "meta-yocto", "giturl": "git://git.yoctoproject.org/poky.git", "dirpath": "meta-yocto", "commit": "HEAD"}),
-    type('poky_info', (object,), { "name": "meta-yocto-bsp", "giturl": "git://git.yoctoproject.org/poky.git", "dirpath": "meta-yocto-bsp", "commit": "HEAD"}),
-    ]
-
-
-
-# we have an abstract test class designed to ensure that the controllers use a single interface
-# specific controller tests only need to override the _getBuildEnvironment() method
-
-test_sourcedir = os.getenv("TTS_SOURCE_DIR")
-test_builddir = os.getenv("TTS_BUILD_DIR")
-test_address = os.getenv("TTS_TEST_ADDRESS", "localhost")
-
-if test_sourcedir == None or test_builddir == None or test_address == None:
-    raise Exception("Please set TTTS_SOURCE_DIR, TTS_BUILD_DIR and TTS_TEST_ADDRESS")
-
-# The bb server will expect a toaster-pre.conf file to exist. If it doesn't exit then we make
-# an empty one here.
-open(test_builddir + 'conf/toaster-pre.conf', 'a').close()
-
-class BEControllerTests(object):
-
-    def _serverForceStop(self, bc):
-        err = bc._shellcmd("netstat  -tapn 2>/dev/null | grep 8200 | awk '{print $7}' | sort -fu | cut -d \"/\" -f 1 | grep -v -- - | tee /dev/fd/2 | xargs -r kill")
-        self.assertTrue(err == '', "bitbake server pid %s not stopped" % err)
-
-    def test_serverStartAndStop(self):
-        obe =  self._getBuildEnvironment()
-        bc = self._getBEController(obe)
-        try:
-            # setting layers, skip any layer info
-            bc.setLayers(BITBAKE_LAYER, POKY_LAYERS)
-        except NotImplementedError:
-            print("Test skipped due to command not implemented yet")
-            return True
-        # We are ok with the exception as we're handling the git already exists
-        except BuildSetupException:
-            pass
-
-        bc.pokydirname = test_sourcedir
-        bc.islayerset = True
-
-        hostname = test_address.split("@")[-1]
-
-        # test start server and stop
-        bc.startBBServer()
-
-        self.assertFalse(socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex((hostname, int(bc.be.bbport))), "Server not answering")
-
-        self._serverForceStop(bc)
-
-    def test_getBBController(self):
-        obe = self._getBuildEnvironment()
-        bc = self._getBEController(obe)
-        layerSet = False
-        try:
-            # setting layers, skip any layer info
-            layerSet = bc.setLayers(BITBAKE_LAYER, POKY_LAYERS)
-        except NotImplementedError:
-            print("Test skipped due to command not implemented yet")
-            return True
-        # We are ok with the exception as we're handling the git already exists
-        except BuildSetupException:
-            pass
-
-        bc.pokydirname = test_sourcedir
-        bc.islayerset = True
-
-        bbc = bc.getBBController()
-        self.assertTrue(isinstance(bbc, BitbakeController))
-
-        self._serverForceStop(bc)
-
-class LocalhostBEControllerTests(TestCase, BEControllerTests):
-    def __init__(self, *args):
-        super(LocalhostBEControllerTests, self).__init__(*args)
-
-
-    def _getBuildEnvironment(self):
-        return BuildEnvironment.objects.create(
-                lock = BuildEnvironment.LOCK_FREE,
-                betype = BuildEnvironment.TYPE_LOCAL,
-                address = test_address,
-                sourcedir = test_sourcedir,
-                builddir = test_builddir )
-
-    def _getBEController(self, obe):
-        return LocalhostBEController(obe)
-
-class RunBuildsCommandTests(TestCase):
-    def test_bec_select(self):
-        """
-        Tests that we can find and lock a build environment
-        """
-
-        obe = BuildEnvironment.objects.create(lock = BuildEnvironment.LOCK_FREE, betype = BuildEnvironment.TYPE_LOCAL)
-        command = Command()
-        bec = command._selectBuildEnvironment()
-
-        # make sure we select the object we've just built
-        self.assertTrue(bec.be.id == obe.id, "Environment is not properly selected")
-        # we have a locked environment
-        self.assertTrue(bec.be.lock == BuildEnvironment.LOCK_LOCK, "Environment is not locked")
-        # no more selections possible here
-        self.assertRaises(IndexError, command._selectBuildEnvironment)
-
-    def test_br_select(self):
-        from orm.models import Project, Release, BitbakeVersion, Branch
-        p = Project.objects.create_project("test", Release.objects.get_or_create(name = "HEAD", bitbake_version = BitbakeVersion.objects.get_or_create(name="HEAD", branch=Branch.objects.get_or_create(name="HEAD"))[0])[0])
-        obr = BuildRequest.objects.create(state = BuildRequest.REQ_QUEUED, project = p)
-        command = Command()
-        br = command._selectBuildRequest()
-
-        # make sure we select the object we've just built
-        self.assertTrue(obr.id == br.id, "Request is not properly selected")
-        # we have a locked environment
-        self.assertTrue(br.state == BuildRequest.REQ_INPROGRESS, "Request is not updated")
-        # no more selections possible here
-        self.assertRaises(IndexError, command._selectBuildRequest)
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/contrib/README b/import-layers/yocto-poky/bitbake/lib/toaster/contrib/README
deleted file mode 100644
index 46d0ff0..0000000
--- a/import-layers/yocto-poky/bitbake/lib/toaster/contrib/README
+++ /dev/null
@@ -1,6 +0,0 @@
-contrib directory for toaster
-
-This directory holds code that works with Toaster, without being an integral part of the Toaster project.
-It is intended for testing code, testing fixtures, tools for Toaster, etc.
-
-NOTE: This directory is NOT a Python module.
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/README b/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/README
deleted file mode 100644
index 22fa567..0000000
--- a/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/README
+++ /dev/null
@@ -1,41 +0,0 @@
-
-Toaster Testing Framework
-Yocto Project
-
-
-Rationale
-------------
-As Toaster contributions grow with the number of people that contribute code, verifying each patch prior to submitting upstream becomes a hard-to-scale problem for humans. We devised this system in order to run patch-level validation, trying to eliminate common problems from submitted patches, in an automated fashion.
-
-The Toaster Testing Framework is a set of Python scripts that provides an extensible way to write smoke and regression tests that will be run on each patch set sent for review on the toaster mailing list.
-
-
-Usage
-------------
-There are three main executable scripts in this directory.
-    *    runner.py  is designed to be run from the command line. It requires, as mandatory parameter, a branch name on poky-contrib, branch which contains the patches to be tested. The program will auto-discover the available tests residing in this directory by looking for unittest classes, and will run the tests on the branch dumping the output to the standard output. Optionally, it can take parameters inhibiting the branch checkout, or specifying a single test to be run, for debugging purposes.
-    *    launcher.py  is a designed to be run from a crontab or similar scheduling mechanism. It looks up a backlog file containing branches-to-test (named tasks in the source code), select the first one in FIFO manner, and launch runner.py on it. It will await for completion, and email the standard output and standard error dumps from the runner.py execution
-    *    recv.py  is an email receiver, designed to be called as a pipe from a .forward file. It is used to monitor a mailing list, for example, and add tasks to the backlog based on review requests coming on the mailing list.
-
-
-Installation
-------------
-As prerequisite, we expect a functioning email system on a machine with Python2.
-
-The broad steps to installation
-* set up the .forward on the receiving email account to pipe to the recv.py file
-* edit config.py and settings.json to alter for local installation settings
-* on email receive, verify backlog.txt to see that the tasks are received and marked for processing
-* execute launcher.py in command line to verify that a test occurs with no problems, and that the outgoing email is delivered
-* add launcher.py
-
-
-
-Contribute
-------------
-What we need are tests. Add your own tests to either tests.py file, or to a new file.
-Use "config.logger" to write logs that will make it to email.
-
-Commonly used code should be going to shellutils, and configuration to config.py.
-
-Contribute code by emailing patches to the list: toaster@yoctoproject.org (membership required)
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/TODO b/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/TODO
deleted file mode 100644
index 1171921..0000000
--- a/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/TODO
+++ /dev/null
@@ -1,9 +0,0 @@
-We need to implement tests:
-
-automated link checker; currently
-$ linkchecker -t 1000 -F csv http://localhost:8000/
-
-integrate the w3c-validation service; currently
-$ python urlcheck.py
-
-
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/config.py b/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/config.py
deleted file mode 100644
index 87b427c..0000000
--- a/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/config.py
+++ /dev/null
@@ -1,98 +0,0 @@
-#!/usr/bin/python
-
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-#
-# Copyright (C) 2015 Alexandru Damian for Intel Corp.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-# This is the configuration/single module for tts
-# everything that would be a global variable goes here
-
-import os, sys, logging
-import socket
-
-LOGDIR = "log"
-SETTINGS_FILE = os.path.join(os.path.dirname(__file__), "settings.json")
-TEST_DIR_NAME = "tts_testdir"
-
-DEBUG = True
-
-OWN_PID = os.getpid()
-
-W3C_VALIDATOR = "http://icarus.local/w3c-validator/check?doctype=HTML5&uri="
-
-TOASTER_PORT = 56789
-
-TESTDIR = None
-
-#we parse the w3c URL to know where to connect
-
-import urlparse
-
-def get_public_ip():
-    temp_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-    parsed_url = urlparse.urlparse("http://icarus.local/w3c-validator/check?doctype=HTML5&uri=")
-    temp_socket.connect((parsed_url.netloc, 80 if parsed_url.port is None else parsed_url.port))
-    public_ip = temp_socket.getsockname()[0]
-    temp_socket.close()
-    return public_ip
-
-TOASTER_BASEURL = "http://%s:%d/" % (get_public_ip(), TOASTER_PORT)
-
-
-OWN_EMAIL_ADDRESS = "Toaster Testing Framework <alexandru.damian@intel.com>"
-REPORT_EMAIL_ADDRESS = "alexandru.damian@intel.com"
-
-# make sure we have the basic logging infrastructure
-
-#pylint: disable=invalid-name
-# we disable the invalid name because the module-level "logger" is used througout bitbake
-logger = logging.getLogger("toastertest")
-__console__ = logging.StreamHandler(sys.stdout)
-__console__.setFormatter(logging.Formatter("%(asctime)s %(levelname)s: %(message)s"))
-logger.addHandler(__console__)
-logger.setLevel(logging.DEBUG)
-
-
-# singleton file names
-LOCKFILE = "/tmp/ttf.lock"
-BACKLOGFILE = os.path.join(os.path.dirname(__file__), "backlog.txt")
-
-# task states
-def enum(*sequential, **named):
-    enums = dict(zip(sequential, range(len(sequential))), **named)
-    reverse = dict((value, key) for key, value in enums.items())
-    enums['reverse_mapping'] = reverse
-    return type('Enum', (), enums)
-
-
-class TASKS(object):
-    #pylint: disable=too-few-public-methods
-    PENDING = "PENDING"
-    INPROGRESS = "INPROGRESS"
-    DONE = "DONE"
-
-    @staticmethod
-    def next_task(task):
-        if task == TASKS.PENDING:
-            return TASKS.INPROGRESS
-        if task == TASKS.INPROGRESS:
-            return TASKS.DONE
-        raise Exception("Invalid next task state for %s" % task)
-
-# TTS specific
-CONTRIB_REPO = "git@git.yoctoproject.org:poky-contrib"
-
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/launcher.py b/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/launcher.py
deleted file mode 100755
index e5794c1..0000000
--- a/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/launcher.py
+++ /dev/null
@@ -1,101 +0,0 @@
-#!/usr/bin/python
-
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-#
-# Copyright (C) 2015 Alexandru Damian for Intel Corp.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-# Program to run the next task listed from the backlog.txt; designed to be
-# run from crontab.
-
-from __future__ import print_function
-import sys, os, config, shellutils
-from shellutils import ShellCmdException
-
-# Import smtplib for the actual sending function
-import smtplib
-
-# Import the email modules we'll need
-from email.mime.text import MIMEText
-
-def _take_lockfile():
-    return shellutils.lockfile(shellutils.mk_lock_filename())
-
-
-def read_next_task_by_state(task_state, task_name=None):
-    if not os.path.exists(os.path.join(os.path.dirname(__file__), config.BACKLOGFILE)):
-        return None
-    os.rename(config.BACKLOGFILE, config.BACKLOGFILE + ".tmp")
-    task = None
-    with open(config.BACKLOGFILE + ".tmp", "r") as f_in:
-        with open(config.BACKLOGFILE, "w") as f_out:
-            for line in f_in.readlines():
-                if task is None:
-                    fields = line.strip().split("|", 2)
-                    if fields[1] == task_state:
-                        if task_name is None or task_name == fields[0]:
-                            task = fields[0]
-                            print("Updating %s %s to %s" % (task, task_state, config.TASKS.next_task(task_state)))
-                            line = "%s|%s\n" % (task, config.TASKS.next_task(task_state))
-                f_out.write(line)
-    os.remove(config.BACKLOGFILE + ".tmp")
-    return task
-
-def send_report(task_name, plaintext, errtext=None):
-    if errtext is None:
-        msg = MIMEText(plaintext)
-    else:
-        if plaintext is None:
-            plaintext = ""
-        msg = MIMEText("--STDOUT dump--\n\n%s\n\n--STDERR dump--\n\n%s" % (plaintext, errtext))
-
-    msg['Subject'] = "[review-request] %s - smoke test results" % task_name
-    msg['From'] = config.OWN_EMAIL_ADDRESS
-    msg['To'] = config.REPORT_EMAIL_ADDRESS
-
-    smtp_connection = smtplib.SMTP("localhost")
-    smtp_connection.sendmail(config.OWN_EMAIL_ADDRESS, [config.REPORT_EMAIL_ADDRESS], msg.as_string())
-    smtp_connection.quit()
-
-def main():
-    # we don't do anything if we have another instance of us running
-    lock_file = _take_lockfile()
-
-    if lock_file is None:
-        if config.DEBUG:
-            print("Concurrent script in progress, exiting")
-        sys.exit(1)
-
-    next_task = read_next_task_by_state(config.TASKS.PENDING)
-    if next_task is not None:
-        print("Next task is", next_task)
-        errtext = None
-        out = None
-        try:
-            out = shellutils.run_shell_cmd("%s %s" % (os.path.join(os.path.dirname(__file__), "runner.py"), next_task))
-        except ShellCmdException as exc:
-            print("Failed while running the test runner: %s", exc)
-            errtext = exc.__str__()
-        send_report(next_task, out, errtext)
-        read_next_task_by_state(config.TASKS.INPROGRESS, next_task)
-    else:
-        print("No task")
-
-    shellutils.unlockfile(lock_file)
-
-
-if __name__ == "__main__":
-    main()
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/recv.py b/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/recv.py
deleted file mode 100755
index 07efdac..0000000
--- a/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/recv.py
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/usr/bin/python
-
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-#
-# Copyright (C) 2015 Alexandru Damian for Intel Corp.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-# Program to receive review requests by email and log tasks to backlog.txt
-# Designed to be run by the email system from a .forward file:
-#
-# cat .forward
-# |[full/path]/recv.py
-
-from __future__ import print_function
-import sys, config, shellutils
-
-from email.parser import Parser
-
-def recv_mail(datastring):
-    headers = Parser().parsestr(datastring)
-    return headers['subject']
-
-def main():
-    lock_file = shellutils.lockfile(shellutils.mk_lock_filename(), retry=True)
-
-    if lock_file is None:
-        if config.DEBUG:
-            print("Concurrent script in progress, exiting")
-        sys.exit(1)
-
-    subject = recv_mail(sys.stdin.read())
-
-    subject_parts = subject.split()
-    if "[review-request]" in subject_parts:
-        task_name = subject_parts[subject_parts.index("[review-request]") + 1]
-        with open(config.BACKLOGFILE, "a") as fout:
-            line = "%s|%s\n" % (task_name, config.TASKS.PENDING)
-            fout.write(line)
-
-    shellutils.unlockfile(lock_file)
-
-if __name__ == "__main__":
-    main()
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/runner.py b/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/runner.py
deleted file mode 100755
index d01386a..0000000
--- a/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/runner.py
+++ /dev/null
@@ -1,222 +0,0 @@
-#!/usr/bin/python
-
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-#
-# Copyright (C) 2015 Alexandru Damian for Intel Corp.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-
-# This is the main test execution controller. It is designed to be run
-# manually from the command line, or to be called from a different program
-# that schedules test execution.
-#
-# Execute   runner.py -h   for help.
-
-
-
-from __future__ import print_function
-import sys, os
-import unittest, importlib
-import logging, pprint, json
-import re
-from shellutils import ShellCmdException, mkdirhier, run_shell_cmd
-
-import config
-
-# we also log to a file, in addition to console, because our output is important
-__log_file_name__ = os.path.join(os.path.dirname(__file__), "log/tts_%d.log" % config.OWN_PID)
-mkdirhier(os.path.dirname(__log_file_name__))
-__log_file__ = open(__log_file_name__, "w")
-__file_handler__ = logging.StreamHandler(__log_file__)
-__file_handler__.setFormatter(logging.Formatter("%(asctime)s %(levelname)s: %(message)s"))
-
-config.logger.addHandler(__file_handler__)
-
-# set up log directory
-try:
-    if not os.path.exists(config.LOGDIR):
-        os.mkdir(config.LOGDIR)
-    else:
-        if not os.path.isdir(config.LOGDIR):
-            raise Exception("Expected log dir '%s' is not actually a directory." % config.LOGDIR)
-except OSError as exc:
-    raise exc
-
-# creates the under-test-branch as a separate directory
-def set_up_test_branch(settings, branch_name):
-    testdir = "%s/%s.%d" % (settings['workdir'], config.TEST_DIR_NAME, config.OWN_PID)
-
-    # creates the host dir
-    if os.path.exists(testdir):
-        raise Exception("Test dir '%s'is already there, aborting" % testdir)
-
-    # may raise OSError, is to be handled by the caller
-    os.makedirs(testdir)
-
-
-    # copies over the .git from the localclone
-    run_shell_cmd("cp -a '%s'/.git '%s'" % (settings['localclone'], testdir))
-
-    # add the remote if it doesn't exist
-    crt_remotes = run_shell_cmd("git remote -v", cwd=testdir)
-    remotes = [word for line in crt_remotes.split("\n") for word in line.split()]
-    if not config.CONTRIB_REPO in remotes:
-        remote_name = "tts_contrib"
-        run_shell_cmd("git remote add %s %s" % (remote_name, config.CONTRIB_REPO), cwd=testdir)
-    else:
-        remote_name = remotes[remotes.index(config.CONTRIB_REPO) - 1]
-
-    # do the fetch
-    run_shell_cmd("git fetch %s -p" % remote_name, cwd=testdir)
-
-    # do the checkout
-    run_shell_cmd("git checkout origin/master && git branch -D %s; git checkout %s/%s -b %s && git reset --hard" % (branch_name, remote_name, branch_name, branch_name), cwd=testdir)
-
-    return testdir
-
-
-def __search_for_tests():
-    # we find all classes that can run, and run them
-    tests = []
-    for _, _, files_list in os.walk(os.path.dirname(os.path.abspath(__file__))):
-        for module_file in [f[:-3] for f in files_list if f.endswith(".py") and not f.startswith("__init__")]:
-            config.logger.debug("Inspecting module %s", module_file)
-            current_module = importlib.import_module(module_file)
-            crtclass_names = vars(current_module)
-            for name in crtclass_names:
-                tested_value = crtclass_names[name]
-                if isinstance(tested_value, type(unittest.TestCase)) and issubclass(tested_value, unittest.TestCase):
-                    tests.append((module_file, name))
-        break
-    return tests
-
-
-# boilerplate to self discover tests and run them
-def execute_tests(dir_under_test, testname):
-
-    if testname is not None and "." in testname:
-        tests = []
-        tests.append(tuple(testname.split(".", 2)))
-    else:
-        tests = __search_for_tests()
-
-    # let's move to the directory under test
-    crt_dir = os.getcwd()
-    os.chdir(dir_under_test)
-
-    # execute each module
-    # pylint: disable=broad-except
-    # we disable the broad-except because we want to actually catch all possible exceptions
-    try:
-        # sorting the tests by the numeric order in the class name
-        tests = sorted(tests, key=lambda x: int(re.search(r"[0-9]+", x[1]).group(0)))
-        config.logger.debug("Discovered test clases: %s", pprint.pformat(tests))
-        unittest.installHandler()
-        suite = unittest.TestSuite()
-        loader = unittest.TestLoader()
-        result = unittest.TestResult()
-        result.failfast = True
-        for module_file, test_name in tests:
-            suite.addTest(loader.loadTestsFromName("%s.%s" % (module_file, test_name)))
-        config.logger.info("Running %d test(s)", suite.countTestCases())
-        suite.run(result)
-
-        for error in result.errors:
-            config.logger.error("Exception on test: %s\n%s", error[0],
-                                "\n".join(["-- %s" % x for x in error[1].split("\n")]))
-
-        for failure in result.failures:
-            config.logger.error("Failed test: %s:\n%s\n", failure[0],
-                                "\n".join(["--  %s" % x for x in failure[1].split("\n")]))
-
-        config.logger.info("Test results: %d ran, %d errors, %d failures", result.testsRun, len(result.errors), len(result.failures))
-
-    except Exception as exc:
-        import traceback
-        config.logger.error("Exception while running test. Tracedump: \n%s", traceback.format_exc())
-    finally:
-        os.chdir(crt_dir)
-    return len(result.failures)
-
-# verify that we had a branch-under-test name as parameter
-def validate_args():
-    from optparse import OptionParser
-    parser = OptionParser(usage="usage: %prog [options] branch_under_test")
-
-    parser.add_option("-t", "--test-dir", dest="testdir", default=None, help="Use specified directory to run tests, inhibits the checkout.")
-    parser.add_option("-s", "--single", dest="singletest", default=None, help="Run only the specified test")
-
-    (options, args) = parser.parse_args()
-    if len(args) < 1:
-        raise Exception("Please specify the branch to run on. Use option '-h' when in doubt.")
-    return (options, args)
-
-
-
-
-# load the configuration options
-def read_settings():
-    if not os.path.exists(config.SETTINGS_FILE) or not os.path.isfile(config.SETTINGS_FILE):
-        raise Exception("Config file '%s' cannot be openend" % config.SETTINGS_FILE)
-    return json.loads(open(config.SETTINGS_FILE, "r").read())
-
-
-# cleanup !
-def clean_up(testdir):
-    run_shell_cmd("rm -rf -- '%s'" % testdir)
-
-def dump_info(settings, options, args):
-    """ detailed information about current run configuration, for debugging purposes.
-    """
-    config.logger.debug("Settings:\n%s\nOptions:\n%s\nArguments:\n%s\n", settings, options, args)
-
-def main():
-    (options, args) = validate_args()
-
-    settings = read_settings()
-    need_cleanup = False
-
-    # dump debug info
-    dump_info(settings, options, args)
-
-    testdir = None
-    no_failures = 1
-    try:
-        if options.testdir is not None and os.path.exists(options.testdir):
-            testdir = os.path.abspath(options.testdir)
-            config.logger.info("No checkout, using %s", testdir)
-        else:
-            need_cleanup = True
-            testdir = set_up_test_branch(settings, args[0]) # we expect a branch name as first argument
-
-        config.TESTDIR = testdir    # we let tests know where to run
-
-        # ensure that the test dir only contains no *.pyc leftovers
-        run_shell_cmd("find '%s' -type f -name *.pyc -exec rm {} \\;" % testdir)
-
-        no_failures = execute_tests(testdir, options.singletest)
-
-    except ShellCmdException as exc:
-        import traceback
-        config.logger.error("Error while setting up testing. Traceback: \n%s", traceback.format_exc())
-    finally:
-        if need_cleanup and testdir is not None:
-            clean_up(testdir)
-
-    sys.exit(no_failures)
-
-if __name__ == "__main__":
-    main()
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/settings.json b/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/settings.json
deleted file mode 100644
index bb671ea..0000000
--- a/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/settings.json
+++ /dev/null
@@ -1,5 +0,0 @@
-{
-	"repo": "git@git.yoctoproject.org:poky-contrib",
-	"localclone": "/home/ddalex/ssd/yocto/poky",
-	"workdir": "/home/ddalex/ssd/yocto"
-}
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/shellutils.py b/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/shellutils.py
deleted file mode 100644
index ce64c06..0000000
--- a/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/shellutils.py
+++ /dev/null
@@ -1,141 +0,0 @@
-#!/usr/bin/python
-
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-#
-# Copyright (C) 2015 Alexandru Damian for Intel Corp.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-# Utilities shared by tests and other common bits of code.
-
-import sys, os, subprocess, fcntl, errno
-import config
-from config import logger
-
-
-# License warning; this code is copied from the BitBake project, file bitbake/lib/bb/utils.py
-# The code is originally licensed GPL-2.0, and we redistribute it under still GPL-2.0
-
-# End of copy is marked with #ENDOFCOPY marker
-
-def mkdirhier(directory):
-    """Create a directory like 'mkdir -p', but does not complain if
-    directory already exists like os.makedirs
-    """
-
-    try:
-        os.makedirs(directory)
-    except OSError as exc:
-        if exc.errno != errno.EEXIST:
-            raise exc
-
-def lockfile(name, shared=False, retry=True):
-    """
-    Use the file fn as a lock file, return when the lock has been acquired.
-    Returns a variable to pass to unlockfile().
-    """
-    config.logger.debug("take lockfile %s", name)
-    dirname = os.path.dirname(name)
-    mkdirhier(dirname)
-
-    if not os.access(dirname, os.W_OK):
-        logger.error("Unable to acquire lock '%s', directory is not writable",
-                     name)
-        sys.exit(1)
-
-    operation = fcntl.LOCK_EX
-    if shared:
-        operation = fcntl.LOCK_SH
-    if not retry:
-        operation = operation | fcntl.LOCK_NB
-
-    while True:
-        # If we leave the lockfiles lying around there is no problem
-        # but we should clean up after ourselves. This gives potential
-        # for races though. To work around this, when we acquire the lock
-        # we check the file we locked was still the lock file on disk.
-        # by comparing inode numbers. If they don't match or the lockfile
-        # no longer exists, we start again.
-
-        # This implementation is unfair since the last person to request the
-        # lock is the most likely to win it.
-
-        # pylint: disable=broad-except
-        # we disable the broad-except because we want to actually catch all possible exceptions
-        try:
-            lock_file = open(name, 'a+')
-            fileno = lock_file.fileno()
-            fcntl.flock(fileno, operation)
-            statinfo = os.fstat(fileno)
-            if os.path.exists(lock_file.name):
-                statinfo2 = os.stat(lock_file.name)
-                if statinfo.st_ino == statinfo2.st_ino:
-                    return lock_file
-            lock_file.close()
-        except Exception as exc:
-            try:
-                lock_file.close()
-            except Exception as exc2:
-                config.logger.error("Failed to close the lockfile: %s", exc2)
-            config.logger.error("Failed to acquire the lockfile: %s", exc)
-        if not retry:
-            return None
-
-def unlockfile(lock_file):
-    """
-    Unlock a file locked using lockfile()
-    """
-    try:
-        # If we had a shared lock, we need to promote to exclusive before
-        # removing the lockfile. Attempt this, ignore failures.
-        fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX|fcntl.LOCK_NB)
-        os.unlink(lock_file.name)
-    except (IOError, OSError):
-        pass
-    fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
-    lock_file.close()
-
-#ENDOFCOPY
-
-
-def mk_lock_filename():
-    our_name = os.path.basename(__file__)
-    our_name = ".%s" % ".".join(reversed(our_name.split(".")))
-    return config.LOCKFILE + our_name
-
-
-
-class ShellCmdException(Exception):
-    pass
-
-def run_shell_cmd(command, cwd=None):
-    if cwd is None:
-        cwd = os.getcwd()
-
-    config.logger.debug("_shellcmd: (%s) %s", cwd, command)
-    process = subprocess.Popen(command, cwd=cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    (out, err) = process.communicate()
-    process.wait()
-    if process.returncode:
-        if len(err) == 0:
-            err = "command: %s \n%s" % (command, out)
-        else:
-            err = "command: %s \n%s" % (command, err)
-        config.logger.warning("_shellcmd: error \n%s\n%s", out, err)
-        raise ShellCmdException(err)
-    else:
-        #config.logger.debug("localhostbecontroller: shellcmd success\n%s" % out)
-        return out
-
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/tests.py b/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/tests.py
deleted file mode 100644
index c510ebb..0000000
--- a/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/tests.py
+++ /dev/null
@@ -1,115 +0,0 @@
-#!/usr/bin/python
-
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-#
-# Copyright (C) 2015 Alexandru Damian for Intel Corp.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-
-# Test definitions. The runner will look for and auto-discover the tests
-# no matter what they file are they in, as long as they are in the same directory
-# as this file.
-
-import unittest
-from shellutils import run_shell_cmd, ShellCmdException
-import config
-
-import pexpect
-import sys, os, signal, time
-
-class Test00PyCompilable(unittest.TestCase):
-    ''' Verifies that all Python files are syntactically correct '''
-    def test_compile_file(self):
-        try:
-            run_shell_cmd("find . -name *py -type f -print0 | xargs -0 -n1 -P20 python -m py_compile", config.TESTDIR)
-        except ShellCmdException as exc:
-            self.fail("Error compiling python files: %s" % (exc))
-
-    def test_pylint_file(self):
-        try:
-            run_shell_cmd(r"find . -iname \"*\.py\" -type f -print0 | PYTHONPATH=${PYTHONPATH}:. xargs -r -0 -n1 pylint --load-plugins pylint_django -E --reports=n 2>&1", cwd=config.TESTDIR + "/bitbake/lib/toaster")
-        except ShellCmdException as exc:
-            self.fail("Pylint fails: %s\n" % exc)
-
-class Test01PySystemStart(unittest.TestCase):
-    ''' Attempts to start Toaster, verify that it is succesfull, and stop it '''
-    def setUp(self):
-        run_shell_cmd("bash -c 'rm -f build/*log'")
-
-    def test_start_interactive_mode(self):
-        try:
-            run_shell_cmd("bash -c 'source %s/oe-init-build-env && source toaster start webport=%d && source toaster stop'" % (config.TESTDIR, config.TOASTER_PORT), config.TESTDIR)
-        except ShellCmdException as exc:
-            self.fail("Failed starting interactive mode: %s" % (exc))
-
-    def test_start_managed_mode(self):
-        try:
-            run_shell_cmd("%s/bitbake/bin/toaster webport=%d nobrowser & sleep 10 && curl http://localhost:%d/ && kill -2 %%1" % (config.TESTDIR, config.TOASTER_PORT, config.TOASTER_PORT), config.TESTDIR)
-        except ShellCmdException as exc:
-            self.fail("Failed starting managed mode: %s" % (exc))
-
-class Test02HTML5Compliance(unittest.TestCase):
-    def setUp(self):
-        self.origdir = os.getcwd()
-        self.crtdir = os.path.dirname(config.TESTDIR)
-        self.cleanup_database = False
-        os.chdir(self.crtdir)
-        if not os.path.exists(os.path.join(self.crtdir, "toaster.sqlite")):
-            self.cleanup_database = True
-            run_shell_cmd("%s/bitbake/lib/toaster/manage.py syncdb --noinput" % config.TESTDIR)
-            run_shell_cmd("%s/bitbake/lib/toaster/manage.py migrate orm" % config.TESTDIR)
-            run_shell_cmd("%s/bitbake/lib/toaster/manage.py migrate bldcontrol" % config.TESTDIR)
-            run_shell_cmd("%s/bitbake/lib/toaster/manage.py loadconf %s/meta-yocto/conf/toasterconf.json" % (config.TESTDIR, config.TESTDIR))
-            run_shell_cmd("%s/bitbake/lib/toaster/manage.py lsupdates" % config.TESTDIR)
-
-            setup = pexpect.spawn("%s/bitbake/lib/toaster/manage.py checksettings" % config.TESTDIR)
-            setup.logfile = sys.stdout
-            setup.expect(r".*or type the full path to a different directory: ")
-            setup.sendline('')
-            setup.sendline('')
-            setup.expect(r".*or type the full path to a different directory: ")
-            setup.sendline('')
-            setup.expect(r"Enter your option: ")
-            setup.sendline('0')
-
-        self.child = pexpect.spawn("bash", ["%s/bitbake/bin/toaster" % config.TESTDIR, "webport=%d" % config.TOASTER_PORT, "nobrowser"], cwd=self.crtdir)
-        self.child.logfile = sys.stdout
-        self.child.expect("Toaster is now running. You can stop it with Ctrl-C")
-
-    def test_html5_compliance(self):
-        import urllist, urlcheck
-        results = {}
-        for url in urllist.URLS:
-            results[url] = urlcheck.validate_html5(config.TOASTER_BASEURL + url)
-
-        failed = []
-        for url in results:
-            if results[url][1] != 0:
-                failed.append((url, results[url]))
-
-
-        self.assertTrue(len(failed) == 0, "Not all URLs validate: \n%s " % "\n".join(["".join(str(x)) for x in failed]))
-
-        #(config.TOASTER_BASEURL + url, status, errors, warnings))
-
-    def tearDown(self):
-        while self.child.isalive():
-            self.child.kill(signal.SIGINT)
-            time.sleep(1)
-        os.chdir(self.origdir)
-        toaster_sqlite_path = os.path.join(self.crtdir, "toaster.sqlite")
-        if self.cleanup_database and os.path.exists(toaster_sqlite_path):
-            os.remove(toaster_sqlite_path)
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/toasteruitest/run_toastertests.py b/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/toasteruitest/run_toastertests.py
deleted file mode 100755
index 8ca45a8..0000000
--- a/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/toasteruitest/run_toastertests.py
+++ /dev/null
@@ -1,155 +0,0 @@
-#!/usr/bin/python
-
-# Copyright
-
-# DESCRIPTION
-# This is script for running all selected toaster cases on
-# selected web browsers manifested in toaster_test.cfg.
-
-# 1. How to start toaster in yocto:
-# $ source poky/oe-init-build-env
-# $ source toaster start
-# $ bitbake core-image-minimal
-
-# 2. How to install selenium on Ubuntu:
-# $ sudo apt-get install scrot python-pip
-# $ sudo pip install selenium
-
-# 3. How to install selenium addon in firefox:
-# Download the lastest firefox addon from http://release.seleniumhq.org/selenium-ide/
-# Then install it. You can also install firebug and firepath addon
-
-# 4. How to start writing a new case:
-# All you need to do is to implement the function test_xxx() and  pile it on.
-
-# 5. How to test with Chrome browser
-# Download/install chrome on host
-# Download chromedriver from https://code.google.com/p/chromedriver/downloads/list  according to your host type
-# put chromedriver in PATH, (e.g. /usr/bin/, bear in mind to chmod)
-# For windows host, you may put chromedriver.exe in the same directory as chrome.exe
-
-import unittest, sys, os, platform
-import ConfigParser
-import argparse
-from toaster_automation_test import toaster_cases
-
-
-def get_args_parser():
-    description = "Script that runs toaster auto tests."
-    parser = argparse.ArgumentParser(description=description)
-    parser.add_argument('--run-all-tests', required=False, action="store_true", dest="run_all_tests", default=False,
-                       help='Run all tests.')
-    parser.add_argument('--run-suite', required=False, dest='run_suite', default=False,
-                       help='run suite (defined in cfg file)')
-
-    return parser
-
-
-def get_tests():
-    testslist = []
-
-    prefix = 'toaster_automation_test.toaster_cases'
-
-    for t in dir(toaster_cases):
-        if t.startswith('test_'):
-            testslist.append('.'.join((prefix, t)))
-
-    return testslist
-
-
-def get_tests_from_cfg(suite=None):
-
-    testslist = []
-    config = ConfigParser.SafeConfigParser()
-    config.read('toaster_test.cfg')
-
-    if suite is not None:
-        target_suite = suite.lower()
-
-        # TODO: if suite is valid suite
-
-    else:
-        target_suite = platform.system().lower()
-
-    try:
-        tests_from_cfg = eval(config.get('toaster_test_' + target_suite, 'test_cases'))
-    except:
-        print('Failed to get test cases from cfg file. Make sure the format is correct.')
-        return None
-
-    prefix = 'toaster_automation_test.toaster_cases.test_'
-    for t in tests_from_cfg:
-        testslist.append(prefix + str(t))
-
-    return testslist
-
-def main():
-
-    # In case this script is called from other directory
-    os.chdir(os.path.abspath(sys.path[0]))
-
-    parser = get_args_parser()
-    args = parser.parse_args()
-
-    if args.run_all_tests:
-        testslist = get_tests()
-    elif args.run_suite:
-        testslist = get_tests_from_cfg(args.run_suite)
-        os.environ['TOASTER_SUITE'] = args.run_suite
-    else:
-        testslist = get_tests_from_cfg()
-
-    if not testslist:
-        print('Failed to get test cases.')
-        exit(1)
-
-    suite = unittest.TestSuite()
-    loader = unittest.TestLoader()
-    loader.sortTestMethodsUsing = None
-    runner = unittest.TextTestRunner(verbosity=2, resultclass=buildResultClass(args))
-
-    for test in testslist:
-        try:
-            suite.addTests(loader.loadTestsFromName(test))
-        except:
-            return 1
-
-    result = runner.run(suite)
-
-    if result.wasSuccessful():
-        return 0
-    else:
-        return 1
-
-
-def buildResultClass(args):
-    """Build a Result Class to use in the testcase execution"""
-
-    class StampedResult(unittest.TextTestResult):
-        """
-        Custom TestResult that prints the time when a test starts.  As toaster-auto
-        can take a long time (ie a few hours) to run, timestamps help us understand
-        what tests are taking a long time to execute.
-        """
-        def startTest(self, test):
-            import time
-            self.stream.write(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + " - ")
-            super(StampedResult, self).startTest(test)
-
-    return StampedResult
-
-
-if __name__ == "__main__":
-
-    try:
-        ret = main()
-    except:
-        ret = 1
-        import traceback
-        traceback.print_exc()
-    finally:
-        if os.getenv('TOASTER_SUITE'):
-            del os.environ['TOASTER_SUITE']
-    sys.exit(ret)
-
-
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/toasteruitest/toaster_automation_test.py b/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/toasteruitest/toaster_automation_test.py
deleted file mode 100755
index 1a786fa..0000000
--- a/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/toasteruitest/toaster_automation_test.py
+++ /dev/null
@@ -1,2376 +0,0 @@
-#!/usr/bin/python
-# Copyright
-
-# DESCRIPTION
-# This is toaster automation base class and test cases file
-
-# History:
-# 2015.03.09  inital version
-# 2015.03.23  adding toaster_test.cfg, run_toastertest.py so we can run case by case from outside
-
-# Briefs:
-# This file is comprised of 3 parts:
-# I:   common utils like sorting, getting attribute.. etc
-# II:  base class part, which complies with unittest frame work and
-#      contains class selenium-based functions
-# III: test cases
-#      to add new case: just implement new test_xxx() function in class toaster_cases
-
-# NOTES for cases:
-# case 946:
-# step 6 - 8 needs to be observed using screenshots
-# case 956:
-# step 2 - 3 needs to be run manually
-
-import unittest, time, re, sys, getopt, os, logging, string, errno, exceptions
-import shutil, argparse, ConfigParser, platform, json
-from selenium import webdriver
-from selenium.common.exceptions import NoSuchElementException
-from selenium import selenium
-from selenium.webdriver.common.by import By
-from selenium.webdriver.common.keys import Keys
-from selenium.webdriver.support.ui import Select
-import sqlite3 as sqlite
-
-
-###########################################
-#                                         #
-# PART I: utils stuff                     #
-#                                         #
-###########################################
-
-class Listattr(object):
-    """
-    Set of list attribute. This is used to determine what the list content is.
-    Later on we may add more attributes here.
-    """
-    NULL = "null"
-    NUMBERS = "numbers"
-    STRINGS = "strings"
-    PERCENT = "percentage"
-    SIZE = "size"
-    UNKNOWN = "unknown"
-
-
-def get_log_root_dir():
-    max_depth = 5
-    parent_dir = '../'
-    for number in range(0, max_depth):
-        if os.path.isdir(sys.path[0] + os.sep + (os.pardir + os.sep)*number + 'log'):
-            log_root_dir = os.path.abspath(sys.path[0] + os.sep + (os.pardir + os.sep)*number + 'log')
-            break
-
-    if number == (max_depth - 1):
-        print('No log dir found. Please check')
-        raise Exception
-
-    return log_root_dir
-
-
-def mkdir_p(dir):
-    try:
-        os.makedirs(dir)
-    except OSError as exc:
-        if exc.errno == errno.EEXIST and os.path.isdir(dir):
-            pass
-        else:
-            raise
-
-
-def get_list_attr(testlist):
-    """
-    To determine the list content
-    """
-    if not testlist:
-        return Listattr.NULL
-    listtest = testlist[:]
-    try:
-        listtest.remove('')
-    except ValueError:
-        pass
-    pattern_percent = re.compile(r"^([0-9])+(\.)?([0-9])*%$")
-    pattern_size = re.compile(r"^([0-9])+(\.)?([0-9])*( )*(K)*(M)*(G)*B$")
-    pattern_number = re.compile(r"^([0-9])+(\.)?([0-9])*$")
-    def get_patterned_number(pattern, tlist):
-        count = 0
-        for item in tlist:
-            if re.search(pattern, item):
-                count += 1
-        return count
-    if get_patterned_number(pattern_percent, listtest) == len(listtest):
-        return Listattr.PERCENT
-    elif get_patterned_number(pattern_size, listtest) == len(listtest):
-        return Listattr.SIZE
-    elif get_patterned_number(pattern_number, listtest) == len(listtest):
-        return Listattr.NUMBERS
-    else:
-        return Listattr.STRINGS
-
-
-def is_list_sequenced(testlist):
-    """
-    Function to tell if list is sequenced
-    Currently we may have list made up of: Strings ; numbers ; percentage ; time; size
-    Each has respective way to determine if it's sequenced.
-    """
-    test_list = testlist[:]
-    try:
-        test_list.remove('')
-    except ValueError:
-        pass
-
-    if get_list_attr(testlist) == Listattr.NULL :
-        return True
-
-    elif get_list_attr(testlist) == Listattr.STRINGS :
-        return (sorted(test_list) == test_list)
-
-    elif get_list_attr(testlist) == Listattr.NUMBERS :
-        list_number = []
-        for item in test_list:
-            list_number.append(eval(item))
-        return (sorted(list_number) == list_number)
-
-    elif get_list_attr(testlist) == Listattr.PERCENT :
-        list_number = []
-        for item in test_list:
-            list_number.append(eval(item.strip('%')))
-        return (sorted(list_number) == list_number)
-
-    elif get_list_attr(testlist) == Listattr.SIZE :
-        list_number = []
-        # currently SIZE is splitted by space
-        for item in test_list:
-            if item.split()[1].upper() == "KB":
-                list_number.append(1024 * eval(item.split()[0]))
-            elif item.split()[1].upper() == "MB":
-                list_number.append(1024 * 1024 * eval(item.split()[0]))
-            elif item.split()[1].upper() == "GB":
-                list_number.append(1024 * 1024 * 1024 * eval(item.split()[0]))
-            else:
-                list_number.append(eval(item.split()[0]))
-        return (sorted(list_number) == list_number)
-
-    else:
-        print('Unrecognized list type, please check')
-        return False
-
-
-def is_list_inverted(testlist):
-    """
-    Function to tell if list is inverted
-    Currently we may have list made up of: Strings ; numbers ; percentage ; time; size
-    Each has respective way to determine if it's inverted.
-    """
-    test_list = testlist[:]
-    try:
-        test_list.remove('')
-    except ValueError:
-        pass
-
-    if get_list_attr(testlist) == Listattr.NULL :
-        return True
-
-    elif get_list_attr(testlist) == Listattr.STRINGS :
-        return (sorted(test_list, reverse = True) == test_list)
-
-    elif get_list_attr(testlist) == Listattr.NUMBERS :
-        list_number = []
-        for item in test_list:
-            list_number.append(eval(item))
-        return (sorted(list_number, reverse = True) == list_number)
-
-    elif get_list_attr(testlist) == Listattr.PERCENT :
-        list_number = []
-        for item in test_list:
-            list_number.append(eval(item.strip('%')))
-        return (sorted(list_number, reverse = True) == list_number)
-
-    elif get_list_attr(testlist) == Listattr.SIZE :
-        list_number = []
-        # currently SIZE is splitted by space. such as 0 B; 1 KB; 2 MB
-        for item in test_list:
-            if item.split()[1].upper() == "KB":
-                list_number.append(1024 * eval(item.split()[0]))
-            elif item.split()[1].upper() == "MB":
-                list_number.append(1024 * 1024 * eval(item.split()[0]))
-            elif item.split()[1].upper() == "GB":
-                list_number.append(1024 * 1024 * 1024 * eval(item.split()[0]))
-            else:
-                list_number.append(eval(item.split()[0]))
-        return (sorted(list_number, reverse = True) == list_number)
-
-    else:
-        print('Unrecognized list type, please check')
-        return False
-
-def replace_file_content(filename, item, option):
-    f = open(filename)
-    lines = f.readlines()
-    f.close()
-    output = open(filename, 'w')
-    for line in lines:
-        if line.startswith(item):
-            output.write(item + " = '" + option + "'\n")
-        else:
-            output.write(line)
-    output.close()
-
-def extract_number_from_string(s):
-    """
-    extract the numbers in a string. return type is 'list'
-    """
-    return re.findall(r'([0-9]+)', s)
-
-# Below is decorator derived from toaster backend test code
-class NoParsingFilter(logging.Filter):
-    def filter(self, record):
-        return record.levelno == 100
-
-def LogResults(original_class):
-    orig_method = original_class.run
-
-    from time import strftime, gmtime
-    caller = 'toaster'
-    timestamp = strftime('%Y%m%d%H%M%S',gmtime())
-    logfile = os.path.join(os.getcwd(),'results-'+caller+'.'+timestamp+'.log')
-    linkfile = os.path.join(os.getcwd(),'results-'+caller+'.log')
-
-    #rewrite the run method of unittest.TestCase to add testcase logging
-    def run(self, result, *args, **kws):
-        orig_method(self, result, *args, **kws)
-        passed = True
-        testMethod = getattr(self, self._testMethodName)
-        #if test case is decorated then use it's number, else use it's name
-        try:
-            test_case = testMethod.test_case
-        except AttributeError:
-            test_case = self._testMethodName
-
-        class_name = str(testMethod.im_class).split("'")[1]
-
-        #create custom logging level for filtering.
-        custom_log_level = 100
-        logging.addLevelName(custom_log_level, 'RESULTS')
-
-        def results(self, message, *args, **kws):
-            if self.isEnabledFor(custom_log_level):
-                self.log(custom_log_level, message, *args, **kws)
-        logging.Logger.results = results
-
-        logging.basicConfig(filename=logfile,
-                            filemode='w',
-                            format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
-                            datefmt='%H:%M:%S',
-                            level=custom_log_level)
-        for handler in logging.root.handlers:
-            handler.addFilter(NoParsingFilter())
-        local_log = logging.getLogger(caller)
-
-        #check status of tests and record it
-
-        for (name, msg) in result.errors:
-            if (self._testMethodName == str(name).split(' ')[0]) and (class_name in str(name).split(' ')[1]):
-                local_log.results("Testcase "+str(test_case)+": ERROR")
-                local_log.results("Testcase "+str(test_case)+":\n"+msg)
-                passed = False
-        for (name, msg) in result.failures:
-            if (self._testMethodName == str(name).split(' ')[0]) and (class_name in str(name).split(' ')[1]):
-                local_log.results("Testcase "+str(test_case)+": FAILED")
-                local_log.results("Testcase "+str(test_case)+":\n"+msg)
-                passed = False
-        for (name, msg) in result.skipped:
-            if (self._testMethodName == str(name).split(' ')[0]) and (class_name in str(name).split(' ')[1]):
-                local_log.results("Testcase "+str(test_case)+": SKIPPED")
-                passed = False
-        if passed:
-            local_log.results("Testcase "+str(test_case)+": PASSED")
-
-        # Create symlink to the current log
-        if os.path.exists(linkfile):
-            os.remove(linkfile)
-        os.symlink(logfile, linkfile)
-
-    original_class.run = run
-
-    return original_class
-
-
-###########################################
-#                                         #
-# PART II: base class                     #
-#                                         #
-###########################################
-
-@LogResults
-class toaster_cases_base(unittest.TestCase):
-
-    @classmethod
-    def setUpClass(cls):
-        cls.log = cls.logger_create()
-
-    def setUp(self):
-        self.screenshot_sequence = 1
-        self.verificationErrors = []
-        self.accept_next_alert = True
-        self.host_os = platform.system().lower()
-        if os.getenv('TOASTER_SUITE'):
-            self.target_suite = os.getenv('TOASTER_SUITE')
-        else:
-            self.target_suite = self.host_os
-
-        self.parser = ConfigParser.SafeConfigParser()
-        self.parser.read('toaster_test.cfg')
-        self.base_url = eval(self.parser.get('toaster_test_' + self.target_suite, 'toaster_url'))
-
-        # create log dir . Currently , we put log files in log/tmp. After all
-        # test cases are done, move them to log/$datetime dir
-        self.log_tmp_dir = os.path.abspath(sys.path[0]) + os.sep + 'log' + os.sep + 'tmp'
-        try:
-            mkdir_p(self.log_tmp_dir)
-        except OSError :
-            logging.error("%(asctime)s Cannot create tmp dir under log, please check your privilege")
-        # self.log = self.logger_create()
-        # driver setup
-        self.setup_browser()
-
-    @staticmethod
-    def logger_create():
-        log_file = "toaster-auto-" + time.strftime("%Y%m%d%H%M%S") + ".log"
-        if os.path.exists("toaster-auto.log"): os.remove("toaster-auto.log")
-        os.symlink(log_file, "toaster-auto.log")
-
-        log = logging.getLogger("toaster")
-        log.setLevel(logging.DEBUG)
-
-        fh = logging.FileHandler(filename=log_file, mode='w')
-        fh.setLevel(logging.DEBUG)
-
-        ch = logging.StreamHandler(sys.stdout)
-        ch.setLevel(logging.INFO)
-
-        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
-        fh.setFormatter(formatter)
-        ch.setFormatter(formatter)
-
-        log.addHandler(fh)
-        log.addHandler(ch)
-
-        return log
-
-
-    def setup_browser(self, *browser_path):
-        self.browser = eval(self.parser.get('toaster_test_' + self.target_suite, 'test_browser'))
-        print(self.browser)
-        if self.browser == "firefox":
-            driver = webdriver.Firefox()
-        elif self.browser == "chrome":
-            driver = webdriver.Chrome()
-        elif self.browser == "ie":
-            driver = webdriver.Ie()
-        else:
-            driver = None
-            print("unrecognized browser type, please check")
-        self.driver = driver
-        self.driver.implicitly_wait(30)
-        return self.driver
-
-
-    def save_screenshot(self,  **log_args):
-        """
-        This function is used to save screen either by os interface or selenium interface.
-        How to use:
-        self.save_screenshot(screenshot_type = 'native'/'selenium', log_sub_dir = 'xxx',
-                             append_name = 'stepx')
-        where native means screenshot func provided by OS,
-        selenium means screenshot func provided by selenium webdriver
-        """
-        types = [log_args.get('screenshot_type')]
-        # when no screenshot_type is specified
-        if types == [None]:
-            types = ['native', 'selenium']
-        # normally append_name is used to specify which step..
-        add_name = log_args.get('append_name')
-        if not add_name:
-            add_name = '-'
-        # normally there's no need to specify sub_dir
-        sub_dir = log_args.get('log_sub_dir')
-        if not sub_dir:
-            # use casexxx as sub_dir name
-            sub_dir = 'case' + str(self.case_no)
-        for item in types:
-            log_dir = self.log_tmp_dir + os.sep + sub_dir
-            mkdir_p(log_dir)
-            log_path = log_dir + os.sep +  self.browser + '-' +\
-                    item + '-' + add_name + '-' + str(self.screenshot_sequence) + '.png'
-            if item == 'native':
-                if self.host_os == "linux":
-                    os.system("scrot " + log_path)
-                elif self.host_os=="darwin":
-                    os.system("screencapture -x " + log_path)
-            elif item == 'selenium':
-                self.driver.get_screenshot_as_file(log_path)
-            self.screenshot_sequence += 1
-
-    def browser_delay(self):
-        """
-        currently this is a workaround for some chrome test.
-        Sometimes we need a delay to accomplish some operation.
-        But for firefox, mostly we don't need this.
-        To be discussed
-        """
-        if self.browser == "chrome":
-            time.sleep(1)
-        return
-
-
-# these functions are not contained in WebDriver class..
-    def find_element_by_text(self, string):
-        return self.driver.find_element_by_xpath("//*[text()='" + string + "']")
-
-
-    def find_elements_by_text(self, string):
-        return self.driver.find_elements_by_xpath("//*[text()='" + string + "']")
-
-
-    def find_element_by_text_in_table(self, table_id, text_string):
-        """
-        This is used to search some certain 'text' in certain table
-        """
-        try:
-            table_element = self.get_table_element(table_id)
-            element = table_element.find_element_by_xpath("//*[text()='" + text_string + "']")
-        except NoSuchElementException as e:
-            print('no element found')
-            raise
-        return element
-
-
-    def find_element_by_link_text_in_table(self, table_id, link_text):
-        """
-        Assume there're multiple suitable "find_element_by_link_text".
-        In this circumstance we need to specify "table".
-        """
-        try:
-            table_element = self.get_table_element(table_id)
-            element = table_element.find_element_by_link_text(link_text)
-        except NoSuchElementException as e:
-            print('no element found')
-            raise
-        return element
-
-
-    def find_elements_by_link_text_in_table(self, table_id, link_text):
-        """
-        Search link-text in certain table. This helps to narrow down search area.
-        """
-        try:
-            table_element = self.get_table_element(table_id)
-            element_list = table_element.find_elements_by_link_text(link_text)
-        except NoSuchElementException as e:
-            print('no element found')
-            raise
-        return element_list
-
-
-    def find_element_by_partial_link_text_in_table(self, table_id, link_text):
-        """
-        Search element by partial link text in certain table.
-        """
-        try:
-            table_element = self.get_table_element(table_id)
-            element = table_element.find_element_by_partial_link_text(link_text)
-            return element
-        except NoSuchElementException as e:
-            print('no element found')
-            raise
-
-
-    def find_elements_by_partial_link_text_in_table(self, table_id, link_text):
-        """
-        Assume there're multiple suitable "find_partial_element_by_link_text".
-        """
-        try:
-            table_element = self.get_table_element(table_id)
-            element_list = table_element.find_elements_by_partial_link_text(link_text)
-            return element_list
-        except NoSuchElementException as e:
-            print('no element found')
-            raise
-
-
-    def find_element_by_xpath_in_table(self, table_id, xpath):
-        """
-        This helps to narrow down search area. Especially useful when dealing with pop-up form.
-        """
-        try:
-            table_element = self.get_table_element(table_id)
-            element = table_element.find_element_by_xpath(xpath)
-        except NoSuchElementException as e:
-            print('no element found')
-            raise
-        return element
-
-
-    def find_elements_by_xpath_in_table(self, table_id, xpath):
-        """
-        This helps to narrow down search area. Especially useful when dealing with pop-up form.
-        """
-        try:
-            table_element = self.get_table_element(table_id)
-            element_list = table_element.find_elements_by_xpath(xpath)
-        except NoSuchElementException as e:
-            print('no elements found')
-            raise
-        return element_list
-
-
-    def shortest_xpath(self, pname, pvalue):
-        return "//*[@" + pname + "='" + pvalue + "']"
-
-
-#usually elements in the same column are with same class name. for instance: class="outcome" .TBD
-    def get_table_column_text(self, attr_name, attr_value):
-        c_xpath = self.shortest_xpath(attr_name, attr_value)
-        elements = self.driver.find_elements_by_xpath(c_xpath)
-        c_list = []
-        for element in elements:
-            c_list.append(element.text)
-        return c_list
-
-
-    def get_table_column_text_by_column_number(self, table_id, column_number):
-        c_xpath = "//*[@id='" + table_id + "']//td[" + str(column_number) + "]"
-        elements = self.driver.find_elements_by_xpath(c_xpath)
-        c_list = []
-        for element in elements:
-            c_list.append(element.text)
-        return c_list
-
-
-    def get_table_head_text(self, *table_id):
-#now table_id is a tuple...
-        if table_id:
-            thead_xpath = "//*[@id='" + table_id[0] + "']//thead//th[text()]"
-            elements = self.driver.find_elements_by_xpath(thead_xpath)
-            c_list = []
-            for element in elements:
-                if element.text:
-                    c_list.append(element.text)
-            return c_list
-#default table on page
-        else:
-            return self.driver.find_element_by_xpath("//*/table/thead").text
-
-
-
-    def get_table_element(self, table_id, *coordinate):
-        if len(coordinate) == 0:
-#return whole-table element
-            element_xpath = "//*[@id='" + table_id + "']"
-            try:
-                element = self.driver.find_element_by_xpath(element_xpath)
-            except NoSuchElementException as e:
-                raise
-            return element
-        row = coordinate[0]
-
-        if len(coordinate) == 1:
-#return whole-row element
-            element_xpath = "//*[@id='" + table_id + "']/tbody/tr[" + str(row) + "]"
-            try:
-                element = self.driver.find_element_by_xpath(element_xpath)
-            except NoSuchElementException as e:
-                return False
-            return element
-#now we are looking for an element with specified X and Y
-        column = coordinate[1]
-
-        element_xpath = "//*[@id='" + table_id + "']/tbody/tr[" + str(row) + "]/td[" + str(column) + "]"
-        try:
-            element = self.driver.find_element_by_xpath(element_xpath)
-        except NoSuchElementException as e:
-            return False
-        return element
-
-
-    def get_table_data(self, table_id, row_count, column_count):
-        row = 1
-        Lists = []
-        while row <= row_count:
-            column = 1
-            row_content=[]
-            while column <= column_count:
-                s= "//*[@id='" + table_id + "']/tbody/tr[" + str(row) +"]/td[" + str(column) + "]"
-                v = self.driver.find_element_by_xpath(s).text
-                row_content.append(v)
-                column = column + 1
-                print("row_content=",row_content)
-            Lists.extend(row_content)
-            print(Lists[row-1][0])
-            row = row + 1
-        return Lists
-
-    # The is_xxx_present functions only returns True/False
-    # All the log work is done in test procedure, so we can easily trace back
-    # using logging
-    def is_text_present (self, patterns):
-        for pattern in patterns:
-            if str(pattern) not in self.driver.page_source:
-                print('Text "'+pattern+'" is missing')
-                return False
-        return True
-
-
-    def is_element_present(self, how, what):
-        try:
-            self.driver.find_element(how, what)
-        except NoSuchElementException as e:
-            print('Could not find element '+str(what)+' by ' + str(how))
-            return False
-        return True
-
-
-    def is_alert_present(self):
-        try: self.driver.switch_to_alert()
-        except NoAlertPresentException as e: return False
-        return True
-
-
-    def close_alert_and_get_its_text(self):
-        try:
-            alert = self.driver.switch_to_alert()
-            alert_text = alert.text
-            if self.accept_next_alert:
-                alert.accept()
-            else:
-                alert.dismiss()
-            return alert_text
-        finally: self.accept_next_alert = True
-
-
-    def get_case_number(self):
-        """
-        what case are we running now
-        """
-        funcname = sys._getframe(1).f_code.co_name
-        caseno_str = funcname.strip('test_')
-        try:
-            caseno = int(caseno_str)
-        except ValueError:
-            print("get case number error! please check if func name is test_xxx")
-            return False
-        return caseno
-
-
-    def tearDown(self):
-        self.log.info(' END: CASE %s log \n\n' % str(self.case_no))
-        self.driver.quit()
-        self.assertEqual([], self.verificationErrors)
-
-
-###################################################################
-#                                                                 #
-# PART III: test cases                                            #
-# please refer to                                                 #
-# https://bugzilla.yoctoproject.org/tr_show_case.cgi?case_id=xxx  #
-#                                                                 #
-###################################################################
-
-# Note: to comply with the unittest framework, we call these test_xxx functions
-# from run_toastercases.py to avoid calling setUp() and tearDown() multiple times
-
-
-class toaster_cases(toaster_cases_base):
-        ##############
-        #  CASE 901  #
-        ##############
-    def test_901(self):
-        # the reason why get_case_number is not in setUp function is that
-        # otherwise it returns "setUp" instead of "test_xxx"
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-        self.driver.maximize_window()
-        self.driver.get(self.base_url)
-        # open all columns
-        self.driver.find_element_by_id("edit-columns-button").click()
-        # adding explicitly wait for chromedriver..-_-
-        self.browser_delay()
-        self.driver.find_element_by_id("started_on").click()
-        self.browser_delay()
-        self.driver.find_element_by_id("time").click()
-        self.driver.find_element_by_id("edit-columns-button").click()
-        # dict: {lint text name : actual class name}
-        table_head_dict = {'Outcome':'outcome', 'Recipe':'target', 'Machine':'machine', 'Started on':'started_on', 'Completed on':'completed_on', \
-                'Errors':'errors_no', 'Warnings':'warnings_no', 'Time':'time'}
-        for key in table_head_dict:
-            try:
-                self.driver.find_element_by_link_text(key).click()
-            except Exception as e:
-                self.log.error("%s cannot be found on page" % key)
-                raise
-            column_list = self.get_table_column_text("class", table_head_dict[key])
-            # after 1st click, the list should be either sequenced or inverted, but we don't have a "default order" here
-            # the point is, after another click, it should be another order
-            if is_list_inverted(column_list):
-                self.driver.find_element_by_link_text(key).click()
-                column_list = self.get_table_column_text("class", table_head_dict[key])
-                self.assertTrue(is_list_sequenced(column_list), msg=("%s column not in order" % key))
-            else:
-                self.assertTrue(is_list_sequenced(column_list), msg=("%s column not sequenced" % key))
-                self.driver.find_element_by_link_text(key).click()
-                column_list = self.get_table_column_text("class", table_head_dict[key])
-                self.assertTrue(is_list_inverted(column_list), msg=("%s column not inverted" % key))
-        self.log.info("case passed")
-
-
-        ##############
-        #  CASE 902  #
-        ##############
-    def test_902(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-        self.driver.maximize_window()
-        self.driver.get(self.base_url)
-        # Could add more test patterns here in the future. Also, could search some items other than target column in future..
-        patterns = ["minimal", "sato"]
-        for pattern in patterns:
-            ori_target_column_texts = self.get_table_column_text("class", "target")
-            print(ori_target_column_texts)
-            self.driver.find_element_by_id("search").clear()
-            self.driver.find_element_by_id("search").send_keys(pattern)
-            self.driver.find_element_by_id("search-button").click()
-            new_target_column_texts = self.get_table_column_text("class", "target")
-            # if nothing found, we still count it as "pass"
-            if new_target_column_texts:
-                for text in new_target_column_texts:
-                    self.assertTrue(text.find(pattern), msg=("%s item doesn't exist " % pattern))
-            self.driver.find_element_by_css_selector("i.icon-remove").click()
-            target_column_texts = self.get_table_column_text("class", "target")
-            self.assertTrue(ori_target_column_texts == target_column_texts, msg=("builds changed after operations"))
-
-
-        ##############
-        #  CASE 903  #
-        ##############
-    def test_903(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-        self.driver.maximize_window()
-        self.driver.get(self.base_url)
-        # when opening a new page, "started_on" is not displayed by default
-        self.driver.find_element_by_id("edit-columns-button").click()
-        # currently all the delay are for chrome driver -_-
-        self.browser_delay()
-        self.driver.find_element_by_id("started_on").click()
-        self.driver.find_element_by_id("edit-columns-button").click()
-        # step 4
-        items = ["Outcome", "Completed on", "Started on"]
-        for item in items:
-            try:
-                temp_element = self.find_element_by_text_in_table('otable', item)
-                # this is how we find "filter icon" in the same level as temp_element(where "a" means clickable, "i" means icon)
-                self.assertTrue(temp_element.find_element_by_xpath("..//*/a/i[@class='icon-filter filtered']"))
-            except Exception as e:
-                self.assertFalse(True, msg=(" %s cannot be found! %s" % (item, e)))
-                raise
-        # step 5-6
-        temp_element = self.find_element_by_link_text_in_table('otable', 'Outcome')
-        temp_element.find_element_by_xpath("..//*/a/i[@class='icon-filter filtered']").click()
-        self.browser_delay()
-        # the 2nd option, whatever it is
-        self.driver.find_element_by_xpath("(//input[@name='filter'])[2]").click()
-        # click "Apply" button
-        self.driver.find_element_by_xpath("//*[@id='filter_outcome']//*[text()='Apply']").click()
-        # save screen here
-        time.sleep(1)
-        self.save_screenshot(screenshot_type='selenium', append_name='step5')
-        temp_element = self.find_element_by_link_text_in_table('otable', 'Completed on')
-        temp_element.find_element_by_xpath("..//*/a/i[@class='icon-filter filtered']").click()
-        self.browser_delay()
-        self.driver.find_element_by_xpath("//*[@id='filter_completed_on']//*[text()='Apply']").click()
-        # save screen here to compare to previous one
-        # please note that for chrome driver, need a little break before saving
-        # screen here -_-
-        self.browser_delay()
-        self.save_screenshot(screenshot_type='selenium', append_name='step6')
-        self.driver.find_element_by_id("search").clear()
-        self.driver.find_element_by_id("search").send_keys("core-image")
-        self.driver.find_element_by_id("search-button").click()
-
-
-        ##############
-        #  CASE 904  #
-        ##############
-    def test_904(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-        self.driver.maximize_window()
-        self.driver.get(self.base_url)
-        self.driver.find_element_by_partial_link_text("core-image").click()
-        self.driver.find_element_by_link_text("Tasks").click()
-        self.table_name = 'otable'
-        # This is how we find the "default" rows-number!
-        rows_displayed = int(Select(self.driver.find_element_by_css_selector("select.pagesize")).first_selected_option.text)
-        print(rows_displayed)
-        self.assertTrue(self.get_table_element(self.table_name, rows_displayed), msg=("not enough rows displayed"))
-        self.assertFalse(self.get_table_element(self.table_name, rows_displayed + 1), \
-                         msg=("more rows displayed than expected"))
-        # Search text box background text is "Search tasks"
-        self.assertTrue(self.driver.find_element_by_xpath("//*[@id='searchform']/*[@placeholder='Search tasks']"),\
-                        msg=("background text doesn't exist"))
-
-        self.driver.find_element_by_id("search").clear()
-        self.driver.find_element_by_id("search").send_keys("busybox")
-        self.driver.find_element_by_id("search-button").click()
-        self.browser_delay()
-        self.save_screenshot(screenshot_type='selenium', append_name='step5')
-        self.driver.find_element_by_css_selector("i.icon-remove").click()
-        # Save screen here
-        self.save_screenshot(screenshot_type='selenium', append_name='step5_2')
-        self.driver.find_element_by_id("edit-columns-button").click()
-        self.driver.find_element_by_id("cpu_used").click()
-        self.driver.find_element_by_id("disk_io").click()
-        self.driver.find_element_by_id("recipe_version").click()
-        self.driver.find_element_by_id("time_taken").click()
-        self.driver.find_element_by_id("edit-columns-button").click()
-        # The operation is the same as case901
-        # dict: {lint text name : actual class name}
-        table_head_dict = {'Order':'order', 'Recipe':'recipe_name', 'Task':'task_name', 'Executed':'executed', \
-                           'Outcome':'outcome', 'Cache attempt':'cache_attempt', 'Time (secs)':'time_taken', 'CPU usage':'cpu_used', \
-                           'Disk I/O (ms)':'disk_io'}
-        for key in table_head_dict:
-        # This is tricky here: we are doing so because there may be more than 1
-        # same-name link_text in one page. So we only find element inside the table
-            self.find_element_by_link_text_in_table(self.table_name, key).click()
-            column_list = self.get_table_column_text("class", table_head_dict[key])
-        # after 1st click, the list should be either sequenced or inverted, but we don't have a "default order" here
-        # the point is, after another click, it should be another order
-        # the first case is special:this means every item in column_list is the same, so
-        # after one click, either sequenced or inverted will be fine
-            if (is_list_inverted(column_list) and is_list_sequenced(column_list)) \
-                or (not column_list) :
-                self.find_element_by_link_text_in_table(self.table_name, key).click()
-                column_list = self.get_table_column_text("class", table_head_dict[key])
-                self.assertTrue(is_list_sequenced(column_list) or is_list_inverted(column_list), \
-                                msg=("%s column not in any order" % key))
-            elif is_list_inverted(column_list):
-                self.find_element_by_link_text_in_table(self.table_name, key).click()
-                column_list = self.get_table_column_text("class", table_head_dict[key])
-                self.assertTrue(is_list_sequenced(column_list), msg=("%s column not in order" % key))
-            else:
-                self.assertTrue(is_list_sequenced(column_list), msg=("%s column not in order" % key))
-                self.find_element_by_link_text_in_table(self.table_name, key).click()
-                column_list = self.get_table_column_text("class", table_head_dict[key])
-                self.assertTrue(is_list_inverted(column_list), msg=("%s column not inverted" % key))
-        # step 8-10
-        # filter dict: {link text name : filter table name in xpath}
-        filter_dict = {'Executed':'filter_executed', 'Outcome':'filter_outcome', 'Cache attempt':'filter_cache_attempt'}
-        for key in filter_dict:
-            temp_element = self.find_element_by_link_text_in_table(self.table_name, key)
-            # find the filter icon besides it.
-            # And here we must have break (1 sec) to get the popup stuff
-            temp_element.find_element_by_xpath("..//*[@class='icon-filter filtered']").click()
-            self.browser_delay()
-            avail_options = self.driver.find_elements_by_xpath("//*[@id='" + filter_dict[key] + "']//*[@name='filter'][not(@disabled)]")
-            for number in range(0, len(avail_options)):
-                avail_options[number].click()
-                self.browser_delay()
-                # click "Apply"
-                self.driver.find_element_by_xpath("//*[@id='" + filter_dict[key]  + "']//*[@type='submit']").click()
-                # insert screen capture here
-                self.browser_delay()
-                self.save_screenshot(screenshot_type='selenium', append_name='step8')
-                # after the last option was clicked, we don't need operation below anymore
-                if number < len(avail_options)-1:
-                     try:
-                        temp_element = self.find_element_by_link_text_in_table(self.table_name, key)
-                        temp_element.find_element_by_xpath("..//*[@class='icon-filter filtered']").click()
-                        avail_options = self.driver.find_elements_by_xpath("//*[@id='" + filter_dict[key] + "']//*[@name='filter'][not(@disabled)]")
-                     except:
-                        print("in exception")
-                        self.find_element_by_text("Show all tasks").click()
-#                        self.driver.find_element_by_xpath("//*[@id='searchform']/button[2]").click()
-                        temp_element = self.find_element_by_link_text_in_table(self.table_name, key)
-                        temp_element.find_element_by_xpath("..//*[@class='icon-filter filtered']").click()
-                        avail_options = self.driver.find_elements_by_xpath("//*[@id='" + filter_dict[key] + "']//*[@name='filter'][not(@disabled)]")
-                     self.browser_delay()
-        # step 11
-        for item in ['order', 'task_name', 'executed', 'outcome', 'recipe_name', 'recipe_version']:
-            try:
-                self.find_element_by_xpath_in_table(self.table_name, "./tbody/tr[1]/*[@class='" + item + "']/a").click()
-            except NoSuchElementException as e:
-            # let it go...
-                print('no item in the colum' + item)
-            # insert screen shot here
-            self.save_screenshot(screenshot_type='selenium', append_name='step11')
-            self.driver.back()
-        # step 12-14
-        # about test_dict: please refer to testcase 904 requirement step 12-14
-        test_dict = {
-            'Time':{
-                'class':'time_taken',
-                'check_head_list':['Recipe', 'Task', 'Executed', 'Outcome', 'Time (secs)'],
-                'check_column_list':['cpu_used', 'cache_attempt', 'disk_io', 'order', 'recipe_version']
-            },
-            'CPU usage':{
-                'class':'cpu_used',
-                'check_head_list':['Recipe', 'Task', 'Executed', 'Outcome', 'CPU usage'],
-                'check_column_list':['cache_attempt', 'disk_io', 'order', 'recipe_version', 'time_taken']
-            },
-            'Disk I/O':{
-                'class':'disk_io',
-                'check_head_list':['Recipe', 'Task', 'Executed', 'Outcome', 'Disk I/O (ms)'],
-                'check_column_list':['cpu_used', 'cache_attempt', 'order', 'recipe_version', 'time_taken']
-            }
-        }
-        for key in test_dict:
-            self.find_element_by_partial_link_text_in_table('nav', 'core-image').click()
-            self.find_element_by_link_text_in_table('nav', key).click()
-            head_list = self.get_table_head_text('otable')
-            for item in test_dict[key]['check_head_list']:
-                self.assertTrue(item in head_list, msg=("%s not in head row" % item))
-            column_list = self.get_table_column_text('class', test_dict[key]['class'])
-            self.assertTrue(is_list_inverted(column_list), msg=("%s column not inverted" % key))
-
-            self.driver.find_element_by_id("edit-columns-button").click()
-            for item2 in test_dict[key]['check_column_list']:
-                self.driver.find_element_by_id(item2).click()
-            self.driver.find_element_by_id("edit-columns-button").click()
-            # TBD: save screen here
-
-
-        ##############
-        #  CASE 906  #
-        ##############
-    def test_906(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-        self.driver.maximize_window()
-        self.driver.get(self.base_url)
-        self.driver.find_element_by_link_text("core-image-minimal").click()
-        self.find_element_by_link_text_in_table('nav', 'Packages').click()
-        # find "bash" in first column (Packages)
-        self.driver.find_element_by_xpath("//*[@id='otable']//td[1]//*[text()='bash']").click()
-        # save sceen here to observe...
-        # step 6
-        self.driver.find_element_by_partial_link_text("Generated files").click()
-        head_list = self.get_table_head_text('otable')
-        for item in ['File', 'Size']:
-            self.assertTrue(item in head_list, msg=("%s not in head row" % item))
-        c_list = self.get_table_column_text('class', 'path')
-        self.assertTrue(is_list_sequenced(c_list), msg=("column not in order"))
-        # step 7
-        self.driver.find_element_by_partial_link_text("Runtime dependencies").click()
-        # save sceen here to observe...
-        # note that here table name is not 'otable'
-        head_list = self.get_table_head_text('dependencies')
-        for item in ['Package', 'Version', 'Size']:
-            self.assertTrue(item in head_list, msg=("%s not in head row" % item))
-        c_list = self.get_table_column_text_by_column_number('dependencies', 1)
-        self.assertTrue(is_list_sequenced(c_list), msg=("list not in order"))
-        texts = ['Size', 'License', 'Recipe', 'Recipe version', 'Layer', \
-                     'Layer commit']
-        self.failUnless(self.is_text_present(texts))
-
-
-        ##############
-        #  CASE 910  #
-        ##############
-    def test_910(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-        image_type="core-image-minimal"
-        test_package1="busybox"
-        test_package2="lib"
-        self.driver.maximize_window()
-        self.driver.get(self.base_url)
-        self.driver.find_element_by_link_text(image_type).click()
-        self.driver.find_element_by_link_text("Recipes").click()
-        self.save_screenshot(screenshot_type='selenium', append_name='step3')
-
-        self.table_name = 'otable'
-        # This is how we find the "default" rows-number!
-        rows_displayed = int(Select(self.driver.find_element_by_css_selector("select.pagesize")).first_selected_option.text)
-        print(rows_displayed)
-        self.assertTrue(self.get_table_element(self.table_name, rows_displayed))
-        self.assertFalse(self.get_table_element(self.table_name, rows_displayed + 1))
-
-        # Check the default table is sorted by Recipe
-        tasks_column_count = len(self.driver.find_elements_by_xpath("/html/body/div[2]/div/div[2]/div[2]/table/tbody/tr/td[1]"))
-        print(tasks_column_count)
-        default_column_list = self.get_table_column_text_by_column_number(self.table_name, 1)
-        #print default_column_list
-
-        self.assertTrue(is_list_sequenced(default_column_list))
-
-        # Search text box background text is "Search recipes"
-        self.assertTrue(self.driver.find_element_by_xpath("//*[@id='searchform']/*[@placeholder='Search recipes']"))
-
-        self.driver.find_element_by_id("search").clear()
-        self.driver.find_element_by_id("search").send_keys(test_package1)
-        self.driver.find_element_by_id("search-button").click()
-        # Save screen here
-        self.save_screenshot(screenshot_type='selenium', append_name='step4')
-        self.driver.find_element_by_css_selector("i.icon-remove").click()
-        self.save_screenshot(screenshot_type='selenium', append_name='step4_2')
-
-        self.driver.find_element_by_id("edit-columns-button").click()
-        self.driver.find_element_by_id("depends_on").click()
-        self.driver.find_element_by_id("layer_version__branch").click()
-        self.driver.find_element_by_id("layer_version__layer__commit").click()
-        self.driver.find_element_by_id("depends_by").click()
-        self.driver.find_element_by_id("edit-columns-button").click()
-
-        self.find_element_by_link_text_in_table(self.table_name, 'Recipe').click()
-        # Check the inverted table by Recipe
-        # Recipe doesn't have class name
-        #inverted_tasks_column_count = len(self.driver.find_elements_by_xpath("/html/body/div[2]/div/div[2]/div[2]/table/tbody/tr/td[1]"))
-        #print inverted_tasks_column_count
-        #inverted_column_list = self.get_table_column_text_by_column_number(self.table_name, 1)
-        #print inverted_column_list
-
-        #self.driver.find_element_by_partial_link_text("zlib").click()
-        #self.driver.back()
-        #self.assertTrue(is_list_inverted(inverted_column_list))
-        #self.find_element_by_link_text_in_table(self.table_name, 'Recipe').click()
-
-        table_head_dict = {'Recipe':'recipe__name', 'Recipe file':'recipe_file', 'Section':'recipe_section', \
-                'License':'recipe_license', 'Layer':'layer_version__layer__name', \
-                'Layer branch':'layer_version__branch'}
-        for key in table_head_dict:
-            self.find_element_by_link_text_in_table(self.table_name, key).click()
-            column_list = self.get_table_column_text("class", table_head_dict[key])
-            if (is_list_inverted(column_list) and is_list_sequenced(column_list)) \
-                    or (not column_list) :
-                self.find_element_by_link_text_in_table(self.table_name, key).click()
-                column_list = self.get_table_column_text("class", table_head_dict[key])
-                self.assertTrue(is_list_sequenced(column_list) or is_list_inverted(column_list))
-                self.driver.find_element_by_partial_link_text("acl").click()
-                self.driver.back()
-                self.assertTrue(is_list_sequenced(column_list) or is_list_inverted(column_list))
-                # Search text box background text is "Search recipes"
-                self.assertTrue(self.driver.find_element_by_xpath("//*[@id='searchform']/*[@placeholder='Search recipes']"))
-                self.driver.find_element_by_id("search").clear()
-                self.driver.find_element_by_id("search").send_keys(test_package2)
-                self.driver.find_element_by_id("search-button").click()
-                column_search_list = self.get_table_column_text("class", table_head_dict[key])
-                self.assertTrue(is_list_sequenced(column_search_list) or is_list_inverted(column_search_list))
-                self.driver.find_element_by_css_selector("i.icon-remove").click()
-            elif is_list_inverted(column_list):
-                self.find_element_by_link_text_in_table(self.table_name, key).click()
-                column_list = self.get_table_column_text("class", table_head_dict[key])
-                self.assertTrue(is_list_sequenced(column_list))
-                self.driver.find_element_by_partial_link_text("acl").click()
-                self.driver.back()
-                self.assertTrue(is_list_sequenced(column_list))
-                # Search text box background text is "Search recipes"
-                self.assertTrue(self.driver.find_element_by_xpath("//*[@id='searchform']/*[@placeholder='Search recipes']"))
-                self.driver.find_element_by_id("search").clear()
-                self.driver.find_element_by_id("search").send_keys(test_package2)
-                self.driver.find_element_by_id("search-button").click()
-                column_search_list = self.get_table_column_text("class", table_head_dict[key])
-                self.assertTrue(is_list_sequenced(column_search_list))
-                self.driver.find_element_by_css_selector("i.icon-remove").click()
-            else:
-                self.assertTrue(is_list_sequenced(column_list),  msg=("list %s not sequenced" % key))
-                self.find_element_by_link_text_in_table(self.table_name, key).click()
-                column_list = self.get_table_column_text("class", table_head_dict[key])
-                self.assertTrue(is_list_inverted(column_list))
-                try:
-                    self.driver.find_element_by_partial_link_text("acl").click()
-                except:
-                    self.driver.find_element_by_partial_link_text("zlib").click()
-                self.driver.back()
-                self.assertTrue(is_list_inverted(column_list))
-                # Search text box background text is "Search recipes"
-                self.assertTrue(self.driver.find_element_by_xpath("//*[@id='searchform']/*[@placeholder='Search recipes']"))
-                self.driver.find_element_by_id("search").clear()
-                self.driver.find_element_by_id("search").send_keys(test_package2)
-                self.driver.find_element_by_id("search-button").click()
-                column_search_list = self.get_table_column_text("class", table_head_dict[key])
-                #print column_search_list
-                self.assertTrue(is_list_inverted(column_search_list))
-                self.driver.find_element_by_css_selector("i.icon-remove").click()
-
-        # Bug 5919
-        for key in table_head_dict:
-            print(key)
-            self.find_element_by_link_text_in_table(self.table_name, key).click()
-            self.driver.find_element_by_id("edit-columns-button").click()
-            self.driver.find_element_by_id(table_head_dict[key]).click()
-            self.driver.find_element_by_id("edit-columns-button").click()
-            self.browser_delay()
-            # After hide the column, the default table should be sorted by Recipe
-            tasks_column_count = len(self.driver.find_elements_by_partial_link_text("acl"))
-            #print tasks_column_count
-            default_column_list = self.get_table_column_text_by_column_number(self.table_name, 1)
-            #print default_column_list
-            self.assertTrue(is_list_sequenced(default_column_list))
-
-        self.driver.find_element_by_id("edit-columns-button").click()
-        self.driver.find_element_by_id("recipe_file").click()
-        self.driver.find_element_by_id("recipe_section").click()
-        self.driver.find_element_by_id("recipe_license").click()
-        self.driver.find_element_by_id("layer_version__layer__name").click()
-        self.driver.find_element_by_id("edit-columns-button").click()
-
-
-        ##############
-        #  CASE 911  #
-        ##############
-    def test_911(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-        self.driver.maximize_window()
-        self.driver.get(self.base_url)
-        self.driver.find_element_by_link_text("core-image-minimal").click()
-        self.find_element_by_link_text_in_table('nav', 'Recipes').click()
-        # step 3-5
-        self.driver.find_element_by_id("search").clear()
-        self.driver.find_element_by_id("search").send_keys("lib")
-        self.driver.find_element_by_id("search-button").click()
-        # save screen here for observation
-        self.save_screenshot(screenshot_type='selenium', append_name='step5')
-        # step 6
-        self.driver.find_element_by_css_selector("i.icon-remove").click()
-        self.driver.find_element_by_id("search").clear()
-        # we deliberately want "no result" here
-        self.driver.find_element_by_id("search").send_keys("no such input")
-        self.driver.find_element_by_id("search-button").click()
-        try:
-            self.find_element_by_text("Show all recipes").click()
-        except:
-            self.fail(msg='Could not identify blank page elements')
-
-        ##############
-        #  CASE 912  #
-        ##############
-    def test_912(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-        self.driver.maximize_window()
-        self.driver.get(self.base_url)
-        self.driver.find_element_by_link_text("core-image-minimal").click()
-        self.find_element_by_link_text_in_table('nav', 'Recipes').click()
-        # step 3
-        head_list = self.get_table_head_text('otable')
-        for item in ['Recipe', 'Recipe version', 'Recipe file', 'Section', 'License', 'Layer']:
-            self.assertTrue(item in head_list, msg=("item %s not in head row" % item))
-        self.driver.find_element_by_id("edit-columns-button").click()
-        self.driver.find_element_by_id("depends_on").click()
-        self.driver.find_element_by_id("layer_version__branch").click()
-        self.driver.find_element_by_id("layer_version__layer__commit").click()
-        self.driver.find_element_by_id("depends_by").click()
-        self.driver.find_element_by_id("edit-columns-button").click()
-        # check if columns selected above is shown
-        check_list = ['Dependencies', 'Layer branch', 'Layer commit', 'Reverse dependencies']
-        head_list = self.get_table_head_text('otable')
-        time.sleep(2)
-        print(head_list)
-        for item in check_list:
-            self.assertTrue(item in head_list, msg=("item %s not in head row" % item))
-        # un-check 'em all
-        self.driver.find_element_by_id("edit-columns-button").click()
-        self.driver.find_element_by_id("depends_on").click()
-        self.driver.find_element_by_id("layer_version__branch").click()
-        self.driver.find_element_by_id("layer_version__layer__commit").click()
-        self.driver.find_element_by_id("depends_by").click()
-        self.driver.find_element_by_id("edit-columns-button").click()
-        # don't exist any more
-        head_list = self.get_table_head_text('otable')
-        for item in check_list:
-            self.assertFalse(item in head_list, msg=("item %s should not be in head row" % item))
-
-
-        ##############
-        #  CASE 913  #
-        ##############
-    def test_913(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-        self.driver.maximize_window()
-        self.driver.get(self.base_url)
-        self.driver.find_element_by_link_text("core-image-minimal").click()
-        self.find_element_by_link_text_in_table('nav', 'Recipes').click()
-        # step 3
-        head_list = self.get_table_head_text('otable')
-        for item in ['Recipe', 'Recipe version', 'Recipe file', 'Section', 'License', 'Layer']:
-            self.assertTrue(item in head_list, msg=("item %s not in head row" % item))
-        # step 4
-        self.driver.find_element_by_id("edit-columns-button").click()
-        # save screen
-        self.browser_delay()
-        self.save_screenshot(screenshot_type='selenium', append_name='step4')
-        self.driver.find_element_by_id("edit-columns-button").click()
-
-
-        ##############
-        #  CASE 914  #
-        ##############
-    def test_914(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-        image_type="core-image-minimal"
-        test_package1="busybox"
-        test_package2="gdbm"
-        test_package3="gettext-native"
-        driver = self.driver
-        driver.maximize_window()
-        driver.get(self.base_url)
-        driver.find_element_by_link_text(image_type).click()
-        driver.find_element_by_link_text("Recipes").click()
-        driver.find_element_by_link_text(test_package1).click()
-
-        self.table_name = 'information'
-
-        tasks_row_count = len(driver.find_elements_by_xpath("//*[@id='"+self.table_name+"']/table/tbody/tr/td[1]"))
-        tasks_column_count = len(driver.find_elements_by_xpath("//*[@id='"+self.table_name+"']/table/tbody/tr[1]/td"))
-        print('rows: '+str(tasks_row_count))
-        print('columns: '+str(tasks_column_count))
-
-        Tasks_column = self.get_table_column_text_by_column_number(self.table_name, 2)
-        print ("Tasks_column=", Tasks_column)
-
-        key_tasks=["do_fetch", "do_unpack",  "do_patch", "do_configure", "do_compile", "do_install", "do_package", "do_build"]
-        i = 0
-        while i < len(key_tasks):
-            if key_tasks[i] not in Tasks_column:
-                print ("Error! Missing key task: %s" % key_tasks[i])
-            else:
-                print ("%s is in tasks" % key_tasks[i])
-            i = i + 1
-
-        if Tasks_column.index(key_tasks[0]) != 0:
-            print ("Error! %s is not in the right position" % key_tasks[0])
-        else:
-            print ("%s is in right position" % key_tasks[0])
-
-        if Tasks_column[-1] != key_tasks[-1]:
-            print ("Error! %s is not in the right position" % key_tasks[-1])
-        else:
-            print ("%s is in right position" % key_tasks[-1])
-
-        driver.find_element_by_partial_link_text("Packages (").click()
-        packages_name = driver.find_element_by_partial_link_text("Packages (").text
-        print(packages_name)
-        packages_num = int(filter(str.isdigit, repr(packages_name)))
-        print(packages_num)
-
-        #switch the table to show more than 10 rows at a time
-        self.driver.find_element_by_xpath("//*[@id='packages-built']/div[1]/div/select").click()
-        Select(driver.find_element_by_xpath("//*[@id='packages-built']/div[1]/div/select")).select_by_value('150')
-        self.driver.find_element_by_xpath("//*[@id='packages-built']/div[1]/div/select").send_keys(Keys.ENTER)
-
-        packages_row_count = len(driver.find_elements_by_xpath("//*[@id='otable']/tbody/tr/td[1]"))
-        print(packages_row_count)
-
-        if packages_num != packages_row_count:
-            print ("Error! The packages number is not correct")
-        else:
-            print ("The packages number is correct")
-
-        driver.find_element_by_partial_link_text("Build dependencies (").click()
-        depends_name = driver.find_element_by_partial_link_text("Build dependencies (").text
-        print(depends_name)
-        depends_num = int(list(filter(str.isdigit, repr(depends_name))))
-        print(depends_num)
-
-        if depends_num == 0:
-            depends_message = repr(driver.find_element_by_css_selector("div.alert.alert-info").text)
-            print(depends_message)
-            if depends_message.find("has no build dependencies.") < 0:
-                print ("Error! The message isn't expected.")
-            else:
-                print ("The message is expected")
-        else:
-            depends_row_count = len(driver.find_elements_by_xpath("//*[@id='dependencies']/table/tbody/tr/td[1]"))
-            print(depends_row_count)
-            if depends_num != depends_row_count:
-                print ("Error! The dependent packages number is not correct")
-            else:
-                print ("The dependent packages number is correct")
-
-        driver.find_element_by_partial_link_text("Reverse build dependencies (").click()
-        rdepends_name = driver.find_element_by_partial_link_text("Reverse build dependencies (").text
-        print(rdepends_name)
-        rdepends_num = int(filter(str.isdigit, repr(rdepends_name)))
-        print(rdepends_num)
-
-        if rdepends_num == 0:
-            rdepends_message = repr(driver.find_element_by_css_selector("#brought-in-by > div.alert.alert-info").text)
-            print(rdepends_message)
-            if rdepends_message.find("has no reverse build dependencies.") < 0:
-                print ("Error! The message isn't expected.")
-            else:
-                print ("The message is expected")
-        else:
-            print ("The reverse dependent packages number is correct")
-
-        driver.find_element_by_link_text("Recipes").click()
-        driver.find_element_by_link_text(test_package2).click()
-        driver.find_element_by_partial_link_text("Packages (").click()
-        driver.find_element_by_partial_link_text("Build dependencies (").click()
-        driver.find_element_by_partial_link_text("Reverse build dependencies (").click()
-
-
-        driver.find_element_by_link_text("Recipes").click()
-        driver.find_element_by_link_text(test_package3).click()
-
-        native_tasks_row_count = len(driver.find_elements_by_xpath("//*[@id='information']/table/tbody/tr/td[1]"))
-        native_tasks_column_count = len(driver.find_elements_by_xpath("//*[@id='information']/table/tbody/tr[1]/td"))
-        print(native_tasks_row_count)
-        print(native_tasks_column_count)
-
-        Native_Tasks_column = self.get_table_column_text_by_column_number(self.table_name, 2)
-        print ("Native_Tasks_column=", Native_Tasks_column)
-
-        native_key_tasks=["do_fetch", "do_unpack",  "do_patch", "do_configure", "do_compile", "do_install", "do_build"]
-        i = 0
-        while i < len(native_key_tasks):
-            if native_key_tasks[i] not in Native_Tasks_column:
-                print ("Error! Missing key task: %s" % native_key_tasks[i])
-            else:
-                print ("%s is in tasks" % native_key_tasks[i])
-            i = i + 1
-
-        if Native_Tasks_column.index(native_key_tasks[0]) != 0:
-            print ("Error! %s is not in the right position" % native_key_tasks[0])
-        else:
-            print ("%s is in right position" % native_key_tasks[0])
-
-        if Native_Tasks_column[-1] != native_key_tasks[-1]:
-            print ("Error! %s is not in the right position" % native_key_tasks[-1])
-        else:
-            print ("%s is in right position" % native_key_tasks[-1])
-
-        driver.find_element_by_partial_link_text("Packages (").click()
-        native_packages_name = driver.find_element_by_partial_link_text("Packages (").text
-        print(native_packages_name)
-        native_packages_num = int(filter(str.isdigit, repr(native_packages_name)))
-        print(native_packages_num)
-
-        if native_packages_num != 0:
-            print ("Error! Native task shouldn't have any packages.")
-        else:
-            native_package_message = repr(driver.find_element_by_css_selector("#packages-built > div.alert.alert-info").text)
-            print(native_package_message)
-            if native_package_message.find("does not build any packages.") < 0:
-                print ("Error! The message for native task isn't expected.")
-            else:
-                print ("The message for native task is expected.")
-
-        driver.find_element_by_partial_link_text("Build dependencies (").click()
-        native_depends_name = driver.find_element_by_partial_link_text("Build dependencies (").text
-        print(native_depends_name)
-        native_depends_num = int(filter(str.isdigit, repr(native_depends_name)))
-        print(native_depends_num)
-
-        native_depends_row_count = len(driver.find_elements_by_xpath("//*[@id='dependencies']/table/tbody/tr/td[1]"))
-        print(native_depends_row_count)
-
-        if native_depends_num != native_depends_row_count:
-            print ("Error! The dependent packages number is not correct")
-        else:
-            print ("The dependent packages number is correct")
-
-        driver.find_element_by_partial_link_text("Reverse build dependencies (").click()
-        native_rdepends_name = driver.find_element_by_partial_link_text("Reverse build dependencies (").text
-        print(native_rdepends_name)
-        native_rdepends_num = int(filter(str.isdigit, repr(native_rdepends_name)))
-        print(native_rdepends_num)
-
-        native_rdepends_row_count = len(driver.find_elements_by_xpath("//*[@id='brought-in-by']/table/tbody/tr/td[1]"))
-        print(native_rdepends_row_count)
-
-        if native_rdepends_num != native_rdepends_row_count:
-            print ("Error! The reverse dependent packages number is not correct")
-        else:
-            print ("The reverse dependent packages number is correct")
-
-        driver.find_element_by_link_text("Recipes").click()
-
-
-        ##############
-        #  CASE 915  #
-        ##############
-    def test_915(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-        self.driver.maximize_window()
-        self.driver.get(self.base_url)
-        self.driver.find_element_by_link_text("core-image-minimal").click()
-        # step 3
-        self.find_element_by_link_text_in_table('nav', 'Configuration').click()
-        self.driver.find_element_by_link_text("BitBake variables").click()
-        # step 4
-        self.driver.find_element_by_id("search").clear()
-        self.driver.find_element_by_id("search").send_keys("lib")
-        self.driver.find_element_by_id("search-button").click()
-        # save screen to see result
-        self.browser_delay()
-        self.save_screenshot(screenshot_type='selenium', append_name='step4')
-        # step 5
-        self.driver.find_element_by_css_selector("i.icon-remove").click()
-        head_list = self.get_table_head_text('otable')
-        print(head_list)
-        print(len(head_list))
-        self.assertTrue(head_list == ['Variable', 'Value', 'Set in file', 'Description'], \
-                        msg=("head row contents wrong"))
-        # step 8
-        # search other string. and click "Variable" to re-sort, check if table
-        # head is still the same
-        self.driver.find_element_by_id("search").clear()
-        self.driver.find_element_by_id("search").send_keys("poky")
-        self.driver.find_element_by_id("search-button").click()
-        self.find_element_by_link_text_in_table('otable', 'Variable').click()
-        head_list = self.get_table_head_text('otable')
-        self.assertTrue(head_list == ['Variable', 'Value', 'Set in file', 'Description'], \
-                        msg=("head row contents wrong"))
-        self.find_element_by_link_text_in_table('otable', 'Variable').click()
-        head_list = self.get_table_head_text('otable')
-        self.assertTrue(head_list == ['Variable', 'Value', 'Set in file', 'Description'], \
-                        msg=("head row contents wrong"))
-
-
-        ##############
-        #  CASE 916  #
-        ##############
-    def test_916(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-        self.driver.maximize_window()
-        self.driver.get(self.base_url)
-        self.driver.find_element_by_link_text("core-image-minimal").click()
-        # step 2-3
-        self.find_element_by_link_text_in_table('nav', 'Configuration').click()
-        self.driver.find_element_by_link_text("BitBake variables").click()
-        variable_list = self.get_table_column_text('class', 'variable_name')
-        self.assertTrue(is_list_sequenced(variable_list), msg=("list not in order"))
-        # step 4
-        self.find_element_by_link_text_in_table('otable', 'Variable').click()
-        variable_list = self.get_table_column_text('class', 'variable_name')
-        self.assertTrue(is_list_inverted(variable_list), msg=("list not inverted"))
-        self.find_element_by_link_text_in_table('otable', 'Variable').click()
-        # step 5
-        # searching won't change the sequentiality
-        self.driver.find_element_by_id("search").clear()
-        self.driver.find_element_by_id("search").send_keys("lib")
-        self.driver.find_element_by_id("search-button").click()
-        variable_list = self.get_table_column_text('class', 'variable_name')
-        self.assertTrue(is_list_sequenced(variable_list), msg=("list not in order"))
-
-
-        ##############
-        #  CASE 923  #
-        ##############
-    def test_923(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-        self.driver.maximize_window()
-        self.driver.get(self.base_url)
-        # Step 2
-        # default sequence in "Completed on" column is inverted
-        c_list = self.get_table_column_text('class', 'completed_on')
-        self.assertTrue(is_list_inverted(c_list), msg=("list not inverted"))
-        # step 3
-        self.driver.find_element_by_id("edit-columns-button").click()
-        self.driver.find_element_by_id("started_on").click()
-        self.driver.find_element_by_id("time").click()
-        self.driver.find_element_by_id("edit-columns-button").click()
-        head_list = self.get_table_head_text('otable')
-        for item in ['Outcome', 'Recipe', 'Machine', 'Started on', 'Completed on', 'Failed tasks', 'Errors', 'Warnings', 'Time', "Image files", "Project"]:
-            self.failUnless(item in head_list, msg=item+' is missing from table head.')
-
-
-        ##############
-        #  CASE 924  #
-        ##############
-    def test_924(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-        self.driver.maximize_window()
-        self.driver.get(self.base_url)
-        # Please refer to case 924 requirement
-        # default sequence in "Completed on" column is inverted
-        c_list = self.get_table_column_text('class', 'completed_on')
-        self.assertTrue(is_list_inverted(c_list), msg=("list not inverted"))
-        # Step 4
-        # click Errors , order in "Completed on" should be disturbed. Then hide
-        # error column to check if order in "Completed on" can be restored
-#THIS TEST IS NO LONGER VALID DUE TO DESIGN CHANGES. LEAVING IN PENDING UPDATES TO DESIGN
-        #self.find_element_by_link_text_in_table('otable', 'Errors').click()
-        #self.driver.find_element_by_id("edit-columns-button").click()
-        #self.driver.find_element_by_id("errors_no").click()
-        #self.driver.find_element_by_id("edit-columns-button").click()
-        # Note: without time.sleep here, there'll be unpredictable error..TBD
-        time.sleep(1)
-        c_list = self.get_table_column_text('class', 'completed_on')
-        self.assertTrue(is_list_inverted(c_list), msg=("list not inverted"))
-
-
-        ##############
-        #  CASE 940  #
-        ##############
-    def test_940(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-        self.driver.maximize_window()
-        self.driver.get(self.base_url)
-        self.driver.find_element_by_link_text("core-image-minimal").click()
-        # Step 2-3
-        self.find_element_by_link_text_in_table('nav', 'Packages').click()
-        check_head_list = ['Package', 'Package version', 'Size', 'Recipe']
-        head_list = self.get_table_head_text('otable')
-        self.assertTrue(head_list == check_head_list, msg=("head row not as expected"))
-        # Step 4
-        # pulldown menu
-        option_ids = ['recipe__layer_version__layer__name', 'recipe__layer_version__branch', \
-                      'recipe__layer_version__layer__commit', 'license', 'recipe__version']
-        self.driver.find_element_by_id("edit-columns-button").click()
-        for item in option_ids:
-            if not self.driver.find_element_by_id(item).is_selected():
-                self.driver.find_element_by_id(item).click()
-        self.driver.find_element_by_id("edit-columns-button").click()
-        # save screen here to observe that 'Package' and 'Package version' is
-        # not selectable
-        self.browser_delay()
-        self.save_screenshot(screenshot_type='selenium', append_name='step4')
-
-
-        ##############
-        #  CASE 941  #
-        ##############
-    def test_941(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-        self.driver.maximize_window()
-        self.driver.get(self.base_url)
-        self.driver.find_element_by_link_text("core-image-minimal").click()
-        # Step 2-3
-        self.find_element_by_link_text_in_table('nav', 'Packages').click()
-        # column -- Package
-        column_list = self.get_table_column_text_by_column_number('otable', 1)
-        self.assertTrue(is_list_sequenced(column_list), msg=("list not in order"))
-        self.find_element_by_link_text_in_table('otable', 'Size').click()
-
-
-        ##############
-        #  CASE 942  #
-        ##############
-    def test_942(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-        self.driver.maximize_window()
-        self.driver.get(self.base_url)
-        self.driver.find_element_by_link_text("core-image-minimal").click()
-        self.driver.find_element_by_link_text("Packages").click()
-        #get initial table header
-        head_list = self.get_table_head_text('otable')
-        #remove the Recipe column from table header
-        self.driver.find_element_by_id("edit-columns-button").click()
-        self.driver.find_element_by_id("recipe__name").click()
-        self.driver.find_element_by_id("edit-columns-button").click()
-        #get modified table header
-        new_head = self.get_table_head_text('otable')
-        self.assertTrue(head_list > new_head)
-
-        ##############
-        #  CASE 943  #
-        ##############
-    def test_943(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-        self.driver.maximize_window()
-        self.driver.get(self.base_url)
-        self.driver.find_element_by_link_text("core-image-minimal").click()
-        self.driver.find_element_by_link_text("Packages").click()
-        #search for the "bash" package -> this should definitely be present
-        self.driver.find_element_by_id("search").clear()
-        self.driver.find_element_by_id("search").send_keys("bash")
-        self.driver.find_element_by_id("search-button").click()
-        #check for the search result message "XX packages found"
-        self.assertTrue(self.is_text_present("packages found"), msg=("no packages found text"))
-
-
-        ##############
-        #  CASE 944  #
-        ##############
-    def test_944(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-        self.driver.maximize_window()
-        self.driver.get(self.base_url)
-        self.driver.find_element_by_link_text("core-image-minimal").click()
-        # step 1: test Recipes page stuff
-        self.driver.find_element_by_link_text("Recipes").click()
-        # for these 3 items, default status is not-checked
-        self.driver.find_element_by_id("edit-columns-button").click()
-        self.driver.find_element_by_id("layer_version__branch").click()
-        self.driver.find_element_by_id("layer_version__layer__commit").click()
-        self.driver.find_element_by_id("edit-columns-button").click()
-        # otable is the recipes table here
-        otable_head_text = self.get_table_head_text('otable')
-        for item in ["Layer", "Layer branch", "Layer commit"]:
-            self.failIf(item not in otable_head_text, msg=item+' not in table head.')
-        # click the fist recipe, whatever it is
-        self.get_table_element("otable", 1, 1).click()
-        self.assertTrue(self.is_text_present(["Layer", "Layer branch", "Layer commit", "Recipe file"]), \
-                        msg=("text not in web page"))
-
-        # step 2: test Packages page stuff. almost same as above
-        self.driver.back()
-        self.browser_delay()
-        self.driver.find_element_by_link_text("Packages").click()
-        self.driver.find_element_by_id("edit-columns-button").click()
-        self.driver.find_element_by_id("recipe__layer_version__layer__name").click()
-        self.driver.find_element_by_id("recipe__layer_version__branch").click()
-        self.driver.find_element_by_id("recipe__layer_version__layer__commit").click()
-        self.driver.find_element_by_id("edit-columns-button").click()
-        otable_head_text = self.get_table_head_text("otable")
-        for item in ["Layer", "Layer branch", "Layer commit"]:
-            self.assertFalse(item not in otable_head_text, msg=("item %s should be in head row" % item))
-        # click the fist recipe, whatever it is
-        self.get_table_element("otable", 1, 1).click()
-        self.assertTrue(self.is_text_present(["Layer", "Layer branch", "Layer commit"]), \
-                        msg=("text not in web page"))
-
-        # step 3: test Packages core-image-minimal(images) stuff. almost same as above. Note when future element-id changes...
-        self.driver.back()
-        self.driver.find_element_by_link_text("core-image-minimal").click()
-        self.driver.find_element_by_id("edit-columns-button").click()
-        self.driver.find_element_by_id("layer_name").click()
-        self.driver.find_element_by_id("layer_branch").click()
-        self.driver.find_element_by_id("layer_commit").click()
-        self.driver.find_element_by_id("edit-columns-button").click()
-        otable_head_text = self.get_table_head_text("otable")
-        for item in ["Layer", "Layer branch", "Layer commit"]:
-            self.assertFalse(item not in otable_head_text, msg=("item %s should be in head row" % item))
-        # click the fist recipe, whatever it is
-        self.get_table_element("otable", 1, 1).click()
-        self.assertTrue(self.is_text_present(["Layer", "Layer branch", "Layer commit"]), \
-                        msg=("text not in web page"))
-
-        # step 4: check Configuration page
-        self.driver.back()
-        self.driver.find_element_by_link_text("Configuration").click()
-        otable_head_text = self.get_table_head_text()
-        self.assertTrue(self.is_text_present(["Layer", "Layer branch", "Layer commit"]), \
-                        msg=("text not in web page"))
-
-
-        ##############
-        #  CASE 945  #
-        ##############
-    def test_945(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-        self.driver.maximize_window()
-        for item in ["Packages", "Recipes", "Tasks"]:
-            self.driver.get(self.base_url)
-            self.driver.find_element_by_link_text("core-image-minimal").click()
-            self.driver.find_element_by_link_text(items).click()
-
-            # this may be page specific. If future page content changes, try to replace it with new xpath
-            xpath_showrows = "/html/body/div[4]/div/div/div[2]/div[2]/div[2]/div/div/div[2]/select"
-            xpath_table = "html/body/div[4]/div/div/div[2]/div[2]/table/tbody"#"id=('otable')/tbody"
-            self.driver.find_element_by_xpath(xpath_showrows).click()
-            rows_displayed = int(self.driver.find_element_by_xpath(xpath_showrows + "/option[2]").text)
-
-            # not sure if this is a Selenium Select bug: If page is not refreshed here, "select(by visible text)" operation will go back to 100-row page
-            # Sure we can use driver.get(url) to refresh page, but since page will vary, we use click link text here
-            self.driver.find_element_by_link_text(items).click()
-            Select(self.driver.find_element_by_css_selector("select.pagesize")).select_by_visible_text(str(rows_displayed))
-            self.failUnless(self.is_element_present(By.XPATH, xpath_table + "/tr[" + str(rows_displayed) +"]"))
-            self.failIf(self.is_element_present(By.XPATH, xpath_table + "/tr[" + str(rows_displayed+1) +"]"))
-
-            # click 1st package, then go back to check if it's still those rows shown.
-            self.driver.find_element_by_xpath(xpath_otable + "/tr[1]/td[1]/a").click()
-            time.sleep(3)
-            self.driver.find_element_by_link_text(item).click()
-            self.assertTrue(self.is_element_present(By.XPATH, xpath_otable + "/tr[" + str(option_tobeselected) +"]"),\
-                            msg=("Row %d should exist" %option_tobeselected))
-            self.assertFalse(self.is_element_present(By.XPATH, xpath_otable + "/tr[" + str(option_tobeselected+1) +"]"),\
-                            msg=("Row %d should not exist" %(option_tobeselected+1)))
-
-
-
-        ##############
-        #  CASE 946  #
-        ##############
-    def test_946(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-        self.driver.maximize_window()
-        self.driver.get(self.base_url)
-        self.driver.find_element_by_link_text("core-image-minimal").click()
-        self.driver.find_element_by_link_text("Configuration").click()
-        # step 3-4
-        check_list = ["Summary", "BitBake variables"]
-        for item in check_list:
-            if not self.is_element_present(how=By.LINK_TEXT, what=item):
-                self.log.error("%s not found" %item)
-        if not self.is_text_present(['Layers', 'Layer', 'Layer branch', 'Layer commit']):
-            self.log.error("text not found")
-        # step 5
-        self.driver.find_element_by_link_text("BitBake variables").click()
-        if not self.is_text_present(['Variable', 'Value', 'Set in file', 'Description']):
-            self.log.error("text not found")
-        # This may be unstable because it's page-specific
-        # step 6: this is how we find filter beside "Set in file"
-        temp_element = self.find_element_by_text_in_table('otable', "Set in file")
-        temp_element.find_element_by_xpath("..//*/a/i[@class='icon-filter filtered']").click()
-        self.browser_delay()
-        self.driver.find_element_by_xpath("(//input[@name='filter'])[3]").click()
-        btns = self.driver.find_elements_by_css_selector("button.btn.btn-primary")
-        for btn in btns:
-            try:
-                btn.click()
-                break
-            except:
-                pass
-        # save screen here
-        self.browser_delay()
-        self.save_screenshot(screenshot_type='selenium', append_name='step6')
-        self.driver.find_element_by_id("edit-columns-button").click()
-        # save screen here
-        # step 7
-        # we should manually check the step 6-8 result using screenshot
-        self.browser_delay()
-        self.save_screenshot(screenshot_type='selenium', append_name='step7')
-        self.driver.find_element_by_id("edit-columns-button").click()
-        # step 9
-        # click the 1st item, no matter what it is
-        self.driver.find_element_by_xpath("//*[@id='otable']/tbody/tr[1]/td[1]/a").click()
-        # give it 1 sec so the pop-up becomes the "active_element"
-        time.sleep(1)
-        element = self.driver.switch_to.active_element
-        check_list = ['Order', 'Configuration file', 'Operation', 'Line number']
-        for item in check_list:
-            if item not in element.text:
-                self.log.error("%s not found" %item)
-        # any better way to close this pop-up? ... TBD
-        element.find_element_by_class_name("close").click()
-        # step 10 : need to manually check "Yocto Manual" in saved screen
-        self.driver.find_element_by_css_selector("i.icon-share.get-info").click()
-        # save screen here
-        time.sleep(5)
-        self.save_screenshot(screenshot_type='native', append_name='step10')
-
-
-        ##############
-        #  CASE 947  #
-        ##############
-    def test_947(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-        self.driver.maximize_window()
-        self.driver.get(self.base_url)
-        self.driver.find_element_by_link_text("core-image-minimal").click()
-        self.find_element_by_link_text_in_table('nav', 'Configuration').click()
-        # step 2
-        self.driver.find_element_by_link_text("BitBake variables").click()
-        # step 3
-        def xpath_option(column_name):
-            # return xpath of options under "Edit columns" button
-            return self.shortest_xpath('id', 'navTab') + self.shortest_xpath('id', 'editcol') \
-                + self.shortest_xpath('id', column_name)
-        self.driver.find_element_by_id('edit-columns-button').click()
-        # by default, option "Description" and "Set in file" were checked
-        self.driver.find_element_by_xpath(xpath_option('description')).click()
-        self.driver.find_element_by_xpath(xpath_option('file')).click()
-        self.driver.find_element_by_id('edit-columns-button').click()
-        check_list = ['Description', 'Set in file']
-        head_list = self.get_table_head_text('otable')
-        for item in check_list:
-            self.assertFalse(item in head_list, msg=("item %s should not be in head row" % item))
-        # check these 2 options and verify again
-        self.driver.find_element_by_id('edit-columns-button').click()
-        self.driver.find_element_by_xpath(xpath_option('description')).click()
-        self.driver.find_element_by_xpath(xpath_option('file')).click()
-        self.driver.find_element_by_id('edit-columns-button').click()
-        head_list = self.get_table_head_text('otable')
-        for item in check_list:
-            self.assertTrue(item in head_list, msg=("item %s not in head row" % item))
-
-
-        ##############
-        #  CASE 948  #
-        ##############
-    def test_948(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-        self.driver.maximize_window()
-        self.driver.get(self.base_url)
-        self.driver.find_element_by_link_text("core-image-minimal").click()
-        self.find_element_by_link_text_in_table('nav', 'Configuration').click()
-        self.driver.find_element_by_link_text("BitBake variables").click()
-        #get number of variables visible by default
-        number_before_search = self.driver.find_element_by_class_name('page-header').text
-        # search for a while...
-        self.driver.find_element_by_id("search").clear()
-        self.driver.find_element_by_id("search").send_keys("BB")
-        self.driver.find_element_by_id("search-button").click()
-        #get number of variables visible after search
-        number_after_search = self.driver.find_element_by_class_name('page-header').text
-        self.assertTrue(number_before_search > number_after_search, msg=("items should be less after search"))
-
-
-        ##############
-        #  CASE 949  #
-        ##############
-    def test_949(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-        self.driver.maximize_window()
-        self.driver.get(self.base_url)
-        self.driver.find_element_by_link_text("core-image-minimal").click()
-        self.find_element_by_link_text_in_table('nav', 'core-image-minimal').click()
-        # step 3
-        try:
-            self.driver.find_element_by_partial_link_text("Packages included")
-            self.driver.find_element_by_partial_link_text("Directory structure")
-        except Exception as e:
-            self.log.error(e)
-            self.assertFalse(True)
-        # step 4
-        head_list = self.get_table_head_text('otable')
-        for item in ['Package', 'Package version', 'Size', 'Dependencies', 'Reverse dependencies', 'Recipe']:
-            self.assertTrue(item in head_list, msg=("item %s not in head row" % item))
-        # step 5-6
-        self.driver.find_element_by_id("edit-columns-button").click()
-        selectable_class = 'checkbox'
-        # minimum-table : means unselectable items
-        unselectable_class = 'checkbox muted'
-        selectable_check_list = ['Dependencies', 'Layer', 'Layer branch', 'Layer commit', \
-                                 'License', 'Recipe', 'Recipe version', 'Reverse dependencies', \
-                                 'Size', 'Size over total (%)']
-        unselectable_check_list = ['Package', 'Package version']
-        selectable_list = list()
-        unselectable_list = list()
-        selectable_elements = self.driver.find_elements_by_xpath("//*[@id='editcol']//*[@class='" + selectable_class + "']")
-        unselectable_elements = self.driver.find_elements_by_xpath("//*[@id='editcol']//*[@class='" + unselectable_class + "']")
-        for element in selectable_elements:
-            selectable_list.append(element.text)
-        for element in unselectable_elements:
-            unselectable_list.append(element.text)
-        # check them
-        for item in selectable_check_list:
-            self.assertTrue(item in selectable_list, msg=("%s not found in dropdown menu" % item))
-        for item in unselectable_check_list:
-            self.assertTrue(item in unselectable_list, msg=("%s not found in dropdown menu" % item))
-        self.driver.find_element_by_id("edit-columns-button").click()
-        # step 7
-        self.driver.find_element_by_partial_link_text("Directory structure").click()
-        head_list = self.get_table_head_text('dirtable')
-        for item in ['Directory / File', 'Symbolic link to', 'Source package', 'Size', 'Permissions', 'Owner', 'Group']:
-            self.assertTrue(item in head_list, msg=("%s not found in Directory structure table head" % item))
-
-        ##############
-        #  CASE 950  #
-        ##############
-    def test_950(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-        self.driver.maximize_window()
-        self.driver.get(self.base_url)
-        # step3&4: so far we're not sure if there's "successful build" or "failed
-        # build".If either of them doesn't exist, we can still go on other steps
-        check_list = ['Configuration', 'Tasks', 'Recipes', 'Packages', 'Time', 'CPU usage', 'Disk I/O']
-        has_successful_build = 1
-        has_failed_build = 1
-        try:
-            pass_icon = self.driver.find_element_by_xpath("//*[@class='icon-ok-sign success']")
-        except Exception:
-            self.log.info("no successful build exists")
-            has_successful_build = 0
-            pass
-        if has_successful_build:
-            pass_icon.click()
-            # save screen here to check if it matches requirement.
-            self.browser_delay()
-            self.save_screenshot(screenshot_type='selenium', append_name='step3_1')
-            for item in check_list:
-                try:
-                    self.find_element_by_link_text_in_table('nav', item)
-                except Exception:
-                    self.assertFalse(True, msg=("link  %s cannot be found in the page" % item))
-            # step 6
-            check_list_2 = ['Packages included', 'Total package size', \
-                      'License manifest', 'Image files']
-            self.assertTrue(self.is_text_present(check_list_2), msg=("text not in web page"))
-            self.driver.back()
-        try:
-            fail_icon = self.driver.find_element_by_xpath("//*[@class='icon-minus-sign error']")
-        except Exception:
-            has_failed_build = 0
-            self.log.info("no failed build exists")
-            pass
-        if has_failed_build:
-            fail_icon.click()
-            # save screen here to check if it matches requirement.
-            self.browser_delay()
-            self.save_screenshot(screenshot_type='selenium', append_name='step3_2')
-            for item in check_list:
-                try:
-                    self.find_element_by_link_text_in_table('nav', item)
-                except Exception:
-                    self.assertFalse(True, msg=("link  %s cannot be found in the page" % item))
-            # step 7 involved
-            check_list_3 = ['Machine', 'Distro', 'Layers', 'Total number of tasks', 'Tasks executed', \
-                      'Tasks not executed', 'Reuse', 'Recipes built', 'Packages built']
-            self.assertTrue(self.is_text_present(check_list_3), msg=("text not in web page"))
-            self.driver.back()
-
-
-        ##############
-        #  CASE 951  #
-        ##############
-    def test_951(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-        self.driver.maximize_window()
-        self.driver.get(self.base_url)
-        # currently test case itself isn't responsible for creating "1 successful and
-        # 1 failed build"
-        has_successful_build = 1
-        has_failed_build = 1
-        try:
-            fail_icon = self.driver.find_element_by_xpath("//*[@class='icon-minus-sign error']")
-        except Exception:
-            has_failed_build = 0
-            self.log.info("no failed build exists")
-            pass
-        # if there's failed build, we can proceed
-        if has_failed_build:
-            self.driver.find_element_by_partial_link_text("error").click()
-            self.driver.back()
-        # not sure if there "must be" some warnings, so here save a screen
-        self.browser_delay()
-        self.save_screenshot(screenshot_type='selenium', append_name='step4')
-
-
-        ##############
-        #  CASE 955  #
-        ##############
-    def test_955(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-        self.driver.maximize_window()
-        self.driver.get(self.base_url)
-        self.log.info(" You should manually create all images before test starts!")
-        # So far the case itself is not responsable for creating all sorts of images.
-        # So assuming they are already there
-        # step 2
-        self.driver.find_element_by_link_text("core-image-minimal").click()
-        # save screen here to see the page component
-
-
-        ##############
-        #  CASE 956  #
-        ##############
-    def test_956(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-        self.driver.maximize_window()
-        self.driver.get(self.base_url)
-        # step 2-3 need to run manually
-        self.log.info("step 2-3: checking the help message when you hover on help icon of target,\
-                       tasks, recipes, packages need to run manually")
-        self.driver.find_element_by_partial_link_text("Manual").click()
-        if not self.is_text_present("Manual"):
-            self.log.error("please check [Toaster manual] link on page")
-            self.failIf(True)
-
-####################################################################################################
-# Starting backend tests ###########################################################################
-####################################################################################################
-
-        ##############
-        #  CASE 1066 #
-        ##############
-    def test_1066(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-        con=sqlite.connect('toaster.sqlite')
-        cursor = con.cursor()
-        query = "select count(name) from orm_project a, auth_user b where a.user_id = b.id and b.username='_anonuser';"
-        cursor.execute(query)
-        data = cursor.fetchone()
-        self.failUnless(data >= 1)
-
-
-        ##############
-        #  CASE 1071 #
-        ##############
-    def test_1071(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-        con=sqlite.connect('toaster.sqlite')
-        cursor = con.cursor()
-        query = "select name from orm_release;"
-        cursor.execute(query)
-        data = cursor.fetchall()
-        for i in range(0,4):
-            data[i] = data[i][0]
-        data.sort()
-        print(data)
-        json_parse = json.loads(open('toasterconf.json').read())
-        json_data = []
-        for i in range (0,4):
-            json_data.append(json_parse['releases'][i]['name'])
-        json_data.sort()
-        print(json_data)
-        self.failUnless(data == json_data)
-
-        ##############
-        #  CASE 1072 #
-        ##############
-    def test_1072(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-        con=sqlite.connect('toaster.sqlite')
-        cursor = con.cursor()
-        query = "select value from orm_toastersetting where name like 'DEFCONF%';"
-        cursor.execute(query)
-        data = cursor.fetchall()
-        for i in range(0,6):
-            data[i] = data[i][0]
-        print(data)
-        json_parse = json.loads(open('toasterconf.json').read())
-        json_data=json_parse['config']
-        json_data = json_data.values()
-        print(json_data)
-        self.failUnless(data == json_data)
-
-
-        ##############
-        #  CASE 1074 #
-        ##############
-    def test_1074(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-        con=sqlite.connect('toaster.sqlite')
-        cursor = con.cursor()
-        query = "select name from orm_layersource;"
-        cursor.execute(query)
-        data = cursor.fetchall()
-        for i in range(0,3):
-            data[i] = data[i][0]
-        print(data)
-        json_parse = json.loads(open('toasterconf.json').read())
-        json_data = []
-        for i in range(0,3):
-            json_data.append(json_parse['layersources'][i]['name'])
-        print(json_data)
-        self.failUnless(set(data) == set(json_data))
-
-        ##############
-        #  CASE 1075 #
-        ##############
-    def test_1075(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-        con=sqlite.connect('toaster.sqlite')
-        cursor = con.cursor()
-        query = "select value from orm_toastersetting where name like 'DEFAULT_RELEASE';"
-        cursor.execute(query)
-        data = cursor.fetchall()
-        data = data[0][0]
-        print(data)
-        json_parse = json.loads(open('toasterconf.json').read())
-        json_data = json_parse['defaultrelease']
-        print(json_data)
-        self.failUnless(set(data) == set(json_data))
-
-        ##############
-        #  CASE 1076 #
-        ##############
-    def test_1076(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-
-        print('Checking branches for "Local Yocto Project"')
-        con=sqlite.connect('toaster.sqlite')
-        cursor = con.cursor()
-        query = "select name from orm_branch where layer_source_id=1;"
-        cursor.execute(query)
-        data = cursor.fetchall()
-        lenght = len(data)
-        try:
-            for i in range(0,lenght):
-                data[i] = data[i][0]
-        except:
-            pass
-        print(data)
-        json_parse = json.loads(open('toasterconf.json').read())
-        json_location = json_parse['layersources'][0]['name']
-        print(json_location)
-        json_data = json_parse['layersources'][0]['branches']
-        print(json_data)
-        self.failUnless(set(data) == set(json_data))
-
-        print('Checking branches for "OpenEmbedded"')
-        con=sqlite.connect('toaster.sqlite')
-        cursor = con.cursor()
-        query = "select name from orm_branch where layer_source_id=3;"
-        cursor.execute(query)
-        data = cursor.fetchall()
-        lenght = len(data)
-        for i in range(0,lenght):
-            data[i] = data[i][0]
-        print(data)
-        json_parse = json.loads(open('toasterconf.json').read())
-        json_location = json_parse['layersources'][1]['name']
-        print(json_location)
-        json_data = json_parse['layersources'][1]['branches']
-        print(json_data)
-        self.failUnless(set(data) == set(json_data))
-
-        print('Checking branches for "Imported layers"')
-        con=sqlite.connect('toaster.sqlite')
-        cursor = con.cursor()
-        query = "select name from orm_branch where layer_source_id=2;"
-        cursor.execute(query)
-        data = cursor.fetchall()
-        lenght = len(data)
-        for i in range(0,lenght):
-            data[i] = data[i][0]
-        print(data)
-        json_parse = json.loads(open('toasterconf.json').read())
-        json_location = json_parse['layersources'][2]['name']
-        print(json_location)
-        json_data = json_parse['layersources'][2]['branches']
-        print(json_data)
-        self.failUnless(set(data) == set(json_data))
-
-
-        ##############
-        #  CASE 1077 #
-        ##############
-    def test_1077(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-        con=sqlite.connect('toaster.sqlite')
-        cursor = con.cursor()
-        query = "select name from orm_bitbakeversion;"
-        cursor.execute(query)
-        data = cursor.fetchall()
-        for i in range(0,4):
-            data[i] = data[i][0]
-        print(data)
-        json_parse = json.loads(open('toasterconf.json').read())
-        json_data = []
-        for i in range(0,4):
-            json_data.append(json_parse['bitbake'][i]['name'])
-        print(json_data)
-        self.failUnless(set(data) == set(json_data))
-
-        ##############
-        #  CASE 1083 #
-        ##############
-    def test_1083(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-        self.driver.maximize_window()
-        self.driver.get(self.base_url)
-        self.driver.find_element_by_id("new-project-button").click()
-        self.driver.find_element_by_id("new-project-name").send_keys("new-test-project")
-        self.driver.find_element_by_id("create-project-button").click()
-        con=sqlite.connect('toaster.sqlite')
-        cursor = con.cursor()
-        query = "select count(name) from orm_project where name = 'new-test-project';"
-        cursor.execute(query)
-        data = cursor.fetchone()
-        print('data: %s' % data)
-        self.failUnless(data >= 1)
-
-        ##############
-        #  CASE 1084 #
-        ##############
-    def test_1084(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-        self.driver.maximize_window()
-        self.driver.get(self.base_url)
-        self.driver.find_element_by_id("new-project-button").click()
-        self.driver.find_element_by_id("new-project-name").send_keys("new-default-project")
-        self.driver.find_element_by_id("create-project-button").click()
-        con=sqlite.connect('toaster.sqlite')
-        cursor = con.cursor()
-        query = "select a.name from orm_release a, orm_project b where a.id = b.release_id and b.name = 'new-default-project' limit 1;"
-        cursor.execute(query)
-        db_data = str(cursor.fetchone()[0])
-        json_parse = json.loads(open('toasterconf.json').read())
-        json_data = str(json_parse['defaultrelease'])
-        self.failUnless(db_data == json_data)
-
-        ##############
-        #  CASE 1088 #
-        ##############
-    def test_1088(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-        self.driver.maximize_window()
-        self.driver.get(self.base_url)
-        self.driver.find_element_by_css_selector("a[href='/toastergui/projects/']").click()
-        self.driver.find_element_by_link_text('new-default-project').click()
-        self.driver.find_element_by_id('project-change-form-toggle').click()
-        self.driver.find_element_by_id('project-name-change-input').clear()
-        self.driver.find_element_by_id('project-name-change-input').send_keys('new-name')
-        self.driver.find_element_by_id('project-name-change-btn').click()
-        con=sqlite.connect('toaster.sqlite')
-        cursor = con.cursor()
-        query = "select count(name) from orm_project where name = 'new-name';"
-        cursor.execute(query)
-        data = cursor.fetchone()[0]
-        self.failUnless(data == 1)
-        #reseting project name
-        self.driver.find_element_by_id('project-change-form-toggle').click()
-        self.driver.find_element_by_id('project-name-change-input').clear()
-        self.driver.find_element_by_id('project-name-change-input').send_keys('new-default-project')
-        self.driver.find_element_by_id('project-name-change-btn').click()
-
-
-        ##############
-        #  CASE 1089 #
-        ##############
-    def test_1089(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-        self.driver.maximize_window()
-        self.driver.get(self.base_url)
-        self.driver.find_element_by_css_selector("a[href='/toastergui/projects/']").click()
-        self.driver.find_element_by_link_text('new-default-project').click()
-        self.driver.find_element_by_id('change-machine-toggle').click()
-        self.driver.find_element_by_id('machine-change-input').clear()
-        self.driver.find_element_by_id('machine-change-input').send_keys('qemuarm64')
-#        self.driver.find_element_by_id('machine-change-input').send_keys(Keys.RETURN)
-        self.driver.find_element_by_id('machine-change-btn').click()
-        con=sqlite.connect('toaster.sqlite')
-        cursor = con.cursor()
-        query = "select count(id) from orm_projectvariable where name like 'machine' and value like 'qemuarm64';"
-        cursor.execute(query)
-        data = cursor.fetchone()[0]
-        self.failUnless(data == 1)
-        #resetting machine to default value
-        self.driver.find_element_by_id('change-machine-toggle').click()
-        self.driver.find_element_by_id('machine-change-input').clear()
-        self.driver.find_element_by_id('machine-change-input').send_keys('qemux86')
-        self.driver.find_element_by_id('machine-change-input').send_keys(Keys.RETURN)
-        self.driver.find_element_by_id('machine-change-btn').click()
-
-        ##############
-        #  CASE 1090 #
-        ##############
-    def test_1090(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-        con=sqlite.connect('toaster.sqlite')
-        cursor = con.cursor()
-        query = "select username from auth_user where is_superuser = 1;"
-        cursor.execute(query)
-        data = cursor.fetchall()
-        try:
-            data = data[0][0]
-        except:
-            pass
-        print(data)
-        self.failUnless(data == 'toaster_admin')
-
-        ##############
-        #  CASE 1091 #
-        ##############
-    def test_1091(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-        self.driver.maximize_window()
-        self.driver.get(self.base_url)
-        self.driver.find_element_by_css_selector("a[href='/toastergui/projects/']").click()
-        self.driver.find_element_by_link_text('new-default-project').click()
-        self.driver.find_element_by_id('release-change-toggle').click()
-        dropdown = self.driver.find_element_by_css_selector('select')
-        for option in dropdown.find_elements_by_tag_name('option'):
-            if option.text == 'Local Yocto Project':
-                option.click()
-        self.driver.find_element_by_id('change-release-btn').click()
-        #wait for the changes to register in the DB
-        time.sleep(1)
-        con=sqlite.connect('toaster.sqlite')
-        cursor = con.cursor()
-        query = "select count(*) from orm_layer_version a, orm_projectlayer b, orm_project c where a.\"commit\"=\"HEAD\" and a.id = b.layercommit_id and b.project_id=c.id and c.name='new-default-project';"
-        cursor.execute(query)
-        data = cursor.fetchone()[0]
-        #resetting release to default
-        self.driver.find_element_by_id('release-change-toggle').click()
-        dropdown = self.driver.find_element_by_css_selector('select')
-        for option in dropdown.find_elements_by_tag_name('option'):
-            if option.text == 'Yocto Project master':
-                option.click()
-        self.driver.find_element_by_id('change-release-btn').click()
-        #wait for the changes to register in the DB
-        time.sleep(1)
-        self.failUnless(data == 3)
-
-        ##############
-        #  CASE 1092 #
-        ##############
-    def test_1092(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-        self.driver.maximize_window()
-        con=sqlite.connect('toaster.sqlite')
-        cursor = con.cursor()
-        query = "select a.name, a.value from orm_projectvariable a, orm_project b where a.project_id = b.id and b.name = 'new-default-project';"
-        cursor.execute(query)
-        data = dict(cursor.fetchall())
-        print(data)
-        default_values = {u'IMAGE_INSTALL_append': u'', u'PACKAGE_CLASSES': u'package_rpm', u'MACHINE': u'qemux86', u'SDKMACHINE': u'x86_64', u'DISTRO': u'poky', u'IMAGE_FSTYPES': u'ext3 jffs2 tar.bz2'}
-        self.failUnless(data == default_values)
-
-        ##############
-        #  CASE 1093 #
-        ##############
-    def test_1093(self):
-        self.case_no = self.get_case_number()
-        self.log.info(' CASE %s log: ' % str(self.case_no))
-
-        #get initial values
-        con=sqlite.connect('toaster.sqlite')
-        cursor = con.cursor()
-        query = "select layercommit_id from orm_projectlayer a, orm_project b where a.project_id=b.id and b.name='new-default-project';"
-        cursor.execute(query)
-        data_initial = cursor.fetchall()
-        print(data_initial)
-
-        self.driver.maximize_window()
-        self.driver.get('localhost:8000')#self.base_url)
-        self.driver.find_element_by_css_selector("a[href='/toastergui/projects/']").click()
-        self.driver.find_element_by_link_text('new-default-project').click()
-        self.driver.find_element_by_id('release-change-toggle').click()
-        dropdown = self.driver.find_element_by_css_selector('select')
-        for option in dropdown.find_elements_by_tag_name('option'):
-            if option.text == 'Local Yocto Project':
-                option.click()
-        self.driver.find_element_by_id('change-release-btn').click()
-        #wait for the changes to register in the DB
-        time.sleep(1)
-
-        #get changed values
-        con=sqlite.connect('toaster.sqlite')
-        cursor = con.cursor()
-        query = "select layercommit_id from orm_projectlayer a, orm_project b where a.project_id=b.id and b.name='new-default-project';"
-        cursor.execute(query)
-        data_changed = cursor.fetchall()
-        print(data_changed)
-
-        #resetting release to default
-        self.driver.find_element_by_id('release-change-toggle').click()
-        dropdown = self.driver.find_element_by_css_selector('select')
-        for option in dropdown.find_elements_by_tag_name('option'):
-            if option.text == 'Yocto Project master':
-                option.click()
-        self.driver.find_element_by_id('change-release-btn').click()
-        #wait for the changes to register in the DB
-        time.sleep(1)
-        self.failUnless(data_initial != data_changed)
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/toasteruitest/toaster_test.cfg b/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/toasteruitest/toaster_test.cfg
deleted file mode 100644
index 685a9ee..0000000
--- a/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/toasteruitest/toaster_test.cfg
+++ /dev/null
@@ -1,25 +0,0 @@
-# Configuration file for toaster_test
-# Sorted by different host type
-
-# test browser could be: firefox; chrome; ie(still under development)
-# logging_level could be: CRITICAL; ERROR; WARNING; INFO; DEBUG; NOTSET
-
-
-[toaster_test_linux]
-toaster_url = 'http://127.0.0.1:8000'
-test_browser = 'firefox'
-test_cases = [946]
-logging_level = 'INFO'
-
-
-[toaster_test_windows]
-toaster_url = 'http://127.0.0.1:8000'
-test_browser = ['ie', 'firefox', 'chrome']
-test_cases = [901, 902, 903]
-logging_level = 'DEBUG'
-
-[toaster_test_darwin]
-toaster_url = 'http://127.0.0.1:8000'
-test_browser = 'firefox'
-test_cases = [901, 902, 903, 904, 906, 910, 911, 912, 913, 914, 915, 916, 923, 924, 940, 941, 942, 943, 944, 945, 946, 947, 948, 949, 950, 951, 955, 956]
-logging_level = 'INFO'
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/urlcheck.py b/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/urlcheck.py
deleted file mode 100644
index 001fcee..0000000
--- a/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/urlcheck.py
+++ /dev/null
@@ -1,53 +0,0 @@
-from __future__ import print_function
-import sys
-
-import httplib2
-import config
-import urllist
-
-
-config.logger.info("Testing %s with %s", config.TOASTER_BASEURL, config.W3C_VALIDATOR)
-
-def validate_html5(url):
-    http_client = httplib2.Http(None)
-    status = "Failed"
-    errors = -1
-    warnings = -1
-
-    urlrequest = config.W3C_VALIDATOR+url
-
-    # pylint: disable=broad-except
-    # we disable the broad-except because we want to actually catch all possible exceptions
-    try:
-        resp, _ = http_client.request(urlrequest, "HEAD")
-        if resp['x-w3c-validator-status'] != "Abort":
-            status = resp['x-w3c-validator-status']
-            errors = int(resp['x-w3c-validator-errors'])
-            warnings = int(resp['x-w3c-validator-warnings'])
-
-            if status == 'Invalid':
-                config.logger.warning("Failed %s is %s\terrors %s warnings %s (check at %s)", url, status, errors, warnings, urlrequest)
-            else:
-                config.logger.debug("OK! %s", url)
-
-    except Exception as exc:
-        config.logger.warning("Failed validation call: %s", exc)
-    return (status, errors, warnings)
-
-
-def print_validation(url):
-    status, errors, warnings = validate_html5(url)
-    config.logger.error("url %s is %s\terrors %s warnings %s (check at %s)", url, status, errors, warnings, config.W3C_VALIDATOR+url)
-
-
-def main():
-    print("Testing %s with %s" % (config.TOASTER_BASEURL, config.W3C_VALIDATOR))
-
-    if len(sys.argv) > 1:
-        print_validation(sys.argv[1])
-    else:
-        for url in urllist.URLS:
-            print_validation(config.TOASTER_BASEURL+url)
-
-if __name__ == "__main__":
-    main()
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/urllist.py b/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/urllist.py
deleted file mode 100644
index 6db9ffc..0000000
--- a/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/urllist.py
+++ /dev/null
@@ -1,39 +0,0 @@
-URLS = [
-    'toastergui/landing/',
-    'toastergui/builds/',
-    'toastergui/build/1',
-    'toastergui/build/1/tasks/',
-    'toastergui/build/1/tasks/1/',
-    'toastergui/build/1/task/1',
-    'toastergui/build/1/recipes/',
-    'toastergui/build/1/recipe/1/active_tab/1',
-    'toastergui/build/1/recipe/1',
-    'toastergui/build/1/recipe_packages/1',
-    'toastergui/build/1/packages/',
-    'toastergui/build/1/package/1',
-    'toastergui/build/1/package_built_dependencies/1',
-    'toastergui/build/1/package_included_detail/1/1',
-    'toastergui/build/1/package_included_dependencies/1/1',
-    'toastergui/build/1/package_included_reverse_dependencies/1/1',
-    'toastergui/build/1/target/1',
-    'toastergui/build/1/target/1/targetpkg',
-    'toastergui/build/1/target/1/dirinfo',
-    'toastergui/build/1/target/1/dirinfo_filepath/_/bin/bash',
-    'toastergui/build/1/configuration',
-    'toastergui/build/1/configvars',
-    'toastergui/build/1/buildtime',
-    'toastergui/build/1/cpuusage',
-    'toastergui/build/1/diskio',
-    'toastergui/build/1/target/1/packagefile/1',
-    'toastergui/newproject/',
-    'toastergui/projects/',
-    'toastergui/project/1',
-    'toastergui/project/1/configuration',
-    'toastergui/project/1/builds/',
-    'toastergui/project/1/layers/',
-    'toastergui/project/1/layer/1',
-    'toastergui/project/1/importlayer',
-    'toastergui/project/1/targets/',
-    'toastergui/project/1/machines/',
-    'toastergui/',
-]
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/orm/fixtures/oe-core.xml b/import-layers/yocto-poky/bitbake/lib/toaster/orm/fixtures/oe-core.xml
index a6c834f..66c3595 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/orm/fixtures/oe-core.xml
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/orm/fixtures/oe-core.xml
@@ -8,21 +8,28 @@
 
   <!-- Bitbake versions which correspond to the metadata release -->
   <object model="orm.bitbakeversion" pk="1">
-    <field type="CharField" name="name">morty</field>
+    <field type="CharField" name="name">pyro</field>
     <field type="CharField" name="giturl">git://git.openembedded.org/bitbake</field>
-    <field type="CharField" name="branch">1.32</field>
+    <field type="CharField" name="branch">1.34</field>
   </object>
   <object model="orm.bitbakeversion" pk="2">
     <field type="CharField" name="name">HEAD</field>
+    <field type="CharField" name="giturl">git://git.openembedded.org/bitbake</field>
+    <field type="CharField" name="branch">HEAD</field>
+  </object>
+  <object model="orm.bitbakeversion" pk="3">
+    <field type="CharField" name="name">master</field>
+    <field type="CharField" name="giturl">git://git.openembedded.org/bitbake</field>
+    <field type="CharField" name="branch">master</field>
   </object>
 
   <!-- Releases available -->
   <object model="orm.release" pk="1">
-    <field type="CharField" name="name">morty</field>
-    <field type="CharField" name="description">Openembedded Morty</field>
+    <field type="CharField" name="name">pyro</field>
+    <field type="CharField" name="description">Openembedded Pyro</field>
     <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">1</field>
-    <field type="CharField" name="branch_name">morty</field>
-    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href=\"http://cgit.openembedded.org/openembedded-core/log/?h=morty\"&gt;OpenEmbedded Morty&lt;/a&gt; branch.</field>
+    <field type="CharField" name="branch_name">pyro</field>
+    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href=\"http://cgit.openembedded.org/openembedded-core/log/?h=pyro\"&gt;OpenEmbedded Pyro&lt;/a&gt; branch.</field>
   </object>
   <object model="orm.release" pk="2">
     <field type="CharField" name="name">local</field>
@@ -31,21 +38,36 @@
     <field type="CharField" name="branch_name">HEAD</field>
     <field type="TextField" name="helptext">Toaster will run your builds with the version of OpenEmbedded that you have cloned or downloaded to your computer.</field>
   </object>
+  <object model="orm.release" pk="3">
+    <field type="CharField" name="name">master</field>
+    <field type="CharField" name="description">OpenEmbedded core master</field>
+    <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">3</field>
+    <field type="CharField" name="branch_name">master</field>
+    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href=\"http://cgit.openembedded.org/openembedded-core/log/\"&gt;OpenEmbedded master&lt;/a&gt; branch.</field>
+  </object>
 
   <!-- Default layers for each release -->
   <object model="orm.releasedefaultlayer" pk="1">
     <field rel="ManyToOneRel" to="orm.release" name="release">1</field>
     <field type="CharField" name="layer_name">openembedded-core</field>
   </object>
-  <object model="orm.releasedefaultlayer" pk="4">
+  <object model="orm.releasedefaultlayer" pk="2">
     <field rel="ManyToOneRel" to="orm.release" name="release">2</field>
     <field type="CharField" name="layer_name">openembedded-core</field>
   </object>
+  <object model="orm.releasedefaultlayer" pk="3">
+    <field rel="ManyToOneRel" to="orm.release" name="release">3</field>
+    <field type="CharField" name="layer_name">openembedded-core</field>
+  </object>
 
-  <!-- TYPE_LOCAL = 0 Layers for the Local release -->
+
+  <!-- Layer for the Local release -->
   <object model="orm.layer" pk="1">
     <field type="CharField" name="name">openembedded-core</field>
     <field type="CharField" name="vcs_url">git://git.openembedded.org/openembedded-core</field>
+    <field type="CharField" name="vcs_web_url">http://cgit.openembedded.org/openembedded-core</field>
+    <field type="CharField" name="vcs_web_tree_base_url">http://cgit.openembedded.org/openembedded-core/tree/%path%?h=%branch%</field>
+    <field type="CharField" name="vcs_web_file_base_url">http://cgit.openembedded.org/openembedded-core/tree/%path%?h=%branch%</field>
   </object>
   <object model="orm.layer_version" pk="1">
     <field rel="ManyToOneRel" to="orm.layer" name="layer">1</field>
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/orm/fixtures/poky.xml b/import-layers/yocto-poky/bitbake/lib/toaster/orm/fixtures/poky.xml
index c192baa..7827aac 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/orm/fixtures/poky.xml
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/orm/fixtures/poky.xml
@@ -8,9 +8,9 @@
 
   <!-- Bitbake versions which correspond to the metadata release -->
   <object model="orm.bitbakeversion" pk="1">
-    <field type="CharField" name="name">morty</field>
+    <field type="CharField" name="name">pyro</field>
     <field type="CharField" name="giturl">git://git.yoctoproject.org/poky</field>
-    <field type="CharField" name="branch">morty</field>
+    <field type="CharField" name="branch">pyro</field>
     <field type="CharField" name="dirpath">bitbake</field>
   </object>
   <object model="orm.bitbakeversion" pk="2">
@@ -19,14 +19,21 @@
     <field type="CharField" name="branch">HEAD</field>
     <field type="CharField" name="dirpath">bitbake</field>
   </object>
+  <object model="orm.bitbakeversion" pk="3">
+    <field type="CharField" name="name">master</field>
+    <field type="CharField" name="giturl">git://git.yoctoproject.org/poky</field>
+    <field type="CharField" name="branch">master</field>
+    <field type="CharField" name="dirpath">bitbake</field>
+  </object>
+
 
   <!-- Releases available -->
   <object model="orm.release" pk="1">
-    <field type="CharField" name="name">morty</field>
-    <field type="CharField" name="description">Yocto Project 2.2 "Morty"</field>
+    <field type="CharField" name="name">pyro</field>
+    <field type="CharField" name="description">Yocto Project 2.3 "Pyro"</field>
     <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">1</field>
-    <field type="CharField" name="branch_name">morty</field>
-    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href="http://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=morty"&gt;Yocto Project Morty branch&lt;/a&gt;.</field>
+    <field type="CharField" name="branch_name">pyro</field>
+    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href="http://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=pyro"&gt;Yocto Project Pyro branch&lt;/a&gt;.</field>
   </object>
   <object model="orm.release" pk="2">
     <field type="CharField" name="name">local</field>
@@ -35,8 +42,15 @@
     <field type="CharField" name="branch_name">HEAD</field>
     <field type="TextField" name="helptext">Toaster will run your builds with the version of the Yocto Project you have cloned or downloaded to your computer.</field>
   </object>
+  <object model="orm.release" pk="3">
+    <field type="CharField" name="name">master</field>
+    <field type="CharField" name="description">Yocto Project master</field>
+    <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">3</field>
+    <field type="CharField" name="branch_name">master</field>
+    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href="http://git.yoctoproject.org/cgit/cgit.cgi/poky/log/"&gt;Yocto Project Master branch&lt;/a&gt;.</field>
+  </object>
 
-  <!-- Default layers for each release -->
+  <!-- Default project layers for each release -->
   <object model="orm.releasedefaultlayer" pk="1">
     <field rel="ManyToOneRel" to="orm.release" name="release">1</field>
     <field type="CharField" name="layer_name">openembedded-core</field>
@@ -61,31 +75,71 @@
     <field rel="ManyToOneRel" to="orm.release" name="release">2</field>
     <field type="CharField" name="layer_name">meta-yocto-bsp</field>
   </object>
+  <object model="orm.releasedefaultlayer" pk="7">
+    <field rel="ManyToOneRel" to="orm.release" name="release">3</field>
+    <field type="CharField" name="layer_name">openembedded-core</field>
+  </object>
+  <object model="orm.releasedefaultlayer" pk="8">
+    <field rel="ManyToOneRel" to="orm.release" name="release">3</field>
+    <field type="CharField" name="layer_name">meta-poky</field>
+  </object>
+  <object model="orm.releasedefaultlayer" pk="9">
+    <field rel="ManyToOneRel" to="orm.release" name="release">3</field>
+    <field type="CharField" name="layer_name">meta-yocto-bsp</field>
+  </object>
 
-  <!-- Layers for the Local release
-       layersource TYPE_LOCAL = 0
+  <!-- Default layers provided by poky
+       openembedded-core
+       meta-poky
+       meta-yocto-bsp
   -->
   <object model="orm.layer" pk="1">
     <field type="CharField" name="name">openembedded-core</field>
     <field type="CharField" name="layer_index_url"></field>
     <field type="CharField" name="vcs_url">git://git.yoctoproject.org/poky</field>
+    <field type="CharField" name="vcs_web_url">http://git.yoctoproject.org/cgit/cgit.cgi/poky</field>
+    <field type="CharField" name="vcs_web_tree_base_url">http://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field>
+    <field type="CharField" name="vcs_web_file_base_url">http://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field>
   </object>
   <object model="orm.layer_version" pk="1">
     <field rel="ManyToOneRel" to="orm.layer" name="layer">1</field>
     <field type="IntegerField" name="layer_source">0</field>
+    <field rel="ManyToOneRel" to="orm.release" name="release">1</field>
+    <field type="CharField" name="branch">pyro</field>
+    <field type="CharField" name="dirpath">meta</field>
+  </object>
+  <object model="orm.layer_version" pk="2">
+    <field rel="ManyToOneRel" to="orm.layer" name="layer">1</field>
+    <field type="IntegerField" name="layer_source">0</field>
     <field rel="ManyToOneRel" to="orm.release" name="release">2</field>
     <field type="CharField" name="branch">HEAD</field>
     <field type="CharField" name="commit">HEAD</field>
     <field type="CharField" name="dirpath">meta</field>
   </object>
-
+  <object model="orm.layer_version" pk="3">
+    <field rel="ManyToOneRel" to="orm.layer" name="layer">1</field>
+    <field type="IntegerField" name="layer_source">0</field>
+    <field rel="ManyToOneRel" to="orm.release" name="release">3</field>
+    <field type="CharField" name="branch">master</field>
+    <field type="CharField" name="dirpath">meta</field>
+  </object>
 
   <object model="orm.layer" pk="2">
     <field type="CharField" name="name">meta-poky</field>
     <field type="CharField" name="layer_index_url"></field>
     <field type="CharField" name="vcs_url">git://git.yoctoproject.org/poky</field>
+    <field type="CharField" name="vcs_web_url">http://git.yoctoproject.org/cgit/cgit.cgi/poky</field>
+    <field type="CharField" name="vcs_web_tree_base_url">http://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field>
+    <field type="CharField" name="vcs_web_file_base_url">http://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field>
   </object>
-  <object model="orm.layer_version" pk="2">
+  <object model="orm.layer_version" pk="4">
+    <field rel="ManyToOneRel" to="orm.layer" name="layer">2</field>
+    <field type="IntegerField" name="layer_source">0</field>
+    <field rel="ManyToOneRel" to="orm.release" name="release">1</field>
+    <field type="CharField" name="branch">pyro</field>
+    <field type="CharField" name="dirpath">meta-poky</field>
+  </object>
+  <object model="orm.layer_version" pk="5">
     <field rel="ManyToOneRel" to="orm.layer" name="layer">2</field>
     <field type="IntegerField" name="layer_source">0</field>
     <field rel="ManyToOneRel" to="orm.release" name="release">2</field>
@@ -93,14 +147,30 @@
     <field type="CharField" name="commit">HEAD</field>
     <field type="CharField" name="dirpath">meta-poky</field>
   </object>
-
+  <object model="orm.layer_version" pk="6">
+    <field rel="ManyToOneRel" to="orm.layer" name="layer">2</field>
+    <field type="IntegerField" name="layer_source">0</field>
+    <field rel="ManyToOneRel" to="orm.release" name="release">3</field>
+    <field type="CharField" name="branch">master</field>
+    <field type="CharField" name="dirpath">meta-poky</field>
+  </object>
 
   <object model="orm.layer" pk="3">
     <field type="CharField" name="name">meta-yocto-bsp</field>
     <field type="CharField" name="layer_index_url"></field>
     <field type="CharField" name="vcs_url">git://git.yoctoproject.org/poky</field>
+    <field type="CharField" name="vcs_web_url">http://git.yoctoproject.org/cgit/cgit.cgi/poky</field>
+    <field type="CharField" name="vcs_web_tree_base_url">http://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field>
+    <field type="CharField" name="vcs_web_file_base_url">http://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field>
   </object>
-  <object model="orm.layer_version" pk="3">
+  <object model="orm.layer_version" pk="7">
+    <field rel="ManyToOneRel" to="orm.layer" name="layer">3</field>
+    <field type="IntegerField" name="layer_source">0</field>
+    <field rel="ManyToOneRel" to="orm.release" name="release">1</field>
+    <field type="CharField" name="branch">pyro</field>
+    <field type="CharField" name="dirpath">meta-yocto-bsp</field>
+  </object>
+  <object model="orm.layer_version" pk="8">
     <field rel="ManyToOneRel" to="orm.layer" name="layer">3</field>
     <field type="IntegerField" name="layer_source">0</field>
     <field rel="ManyToOneRel" to="orm.release" name="release">2</field>
@@ -108,4 +178,11 @@
     <field type="CharField" name="commit">HEAD</field>
     <field type="CharField" name="dirpath">meta-yocto-bsp</field>
   </object>
+  <object model="orm.layer_version" pk="9">
+    <field rel="ManyToOneRel" to="orm.layer" name="layer">3</field>
+    <field type="IntegerField" name="layer_source">0</field>
+    <field rel="ManyToOneRel" to="orm.release" name="release">3</field>
+    <field type="CharField" name="branch">master</field>
+    <field type="CharField" name="dirpath">meta-yocto-bsp</field>
+  </object>
 </django-objects>
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/orm/fixtures/settings.xml b/import-layers/yocto-poky/bitbake/lib/toaster/orm/fixtures/settings.xml
index ee6a202..78c0fdc 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/orm/fixtures/settings.xml
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/orm/fixtures/settings.xml
@@ -4,7 +4,7 @@
   <!-- pk=1 is DISTRO -->
   <object model="orm.toastersetting" pk="2">
     <field type="CharField" name="name">DEFAULT_RELEASE</field>
-    <field type="CharField" name="value">morty</field>
+    <field type="CharField" name="value">master</field>
   </object>
   <object model="orm.toastersetting" pk="3">
     <field type="CharField" name="name">DEFCONF_PACKAGE_CLASSES</field>
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/orm/management/commands/lsupdates.py b/import-layers/yocto-poky/bitbake/lib/toaster/orm/management/commands/lsupdates.py
index 8ff120e..482908d 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/orm/management/commands/lsupdates.py
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/orm/management/commands/lsupdates.py
@@ -90,7 +90,6 @@
             from urlparse import urlparse
 
         proxy_settings = os.environ.get("http_proxy", None)
-        oe_core_layer = 'openembedded-core'
 
         def _get_json_response(apiurl=DEFAULT_LAYERINDEX_SERVER):
             http_progress = Spinner()
@@ -154,41 +153,19 @@
 
         total = len(layers_info)
         for i, li in enumerate(layers_info):
-            # Special case for the openembedded-core layer
-            if li['name'] == oe_core_layer:
-                try:
-                    # If we have an existing openembedded-core for example
-                    # from the toasterconf.json augment the info using the
-                    # layerindex rather than duplicate it
-                    oe_core_l = Layer.objects.get(name=oe_core_layer)
-                    # Take ownership of the layer as now coming from the
-                    # layerindex
-                    oe_core_l.summary = li['summary']
-                    oe_core_l.description = li['description']
-                    oe_core_l.vcs_web_url = li['vcs_web_url']
-                    oe_core_l.vcs_web_tree_base_url = \
-                        li['vcs_web_tree_base_url']
-                    oe_core_l.vcs_web_file_base_url = \
-                        li['vcs_web_file_base_url']
-
-                    oe_core_l.save()
-                    li_layer_id_to_toaster_layer_id[li['id']] = oe_core_l.pk
-                    self.mini_progress("layers", i, total)
-                    continue
-
-                except Layer.DoesNotExist:
-                    pass
-
             try:
-                l, created = Layer.objects.get_or_create(name=li['name'],
-                                                         vcs_url=li['vcs_url'])
+                l, created = Layer.objects.get_or_create(name=li['name'])
                 l.up_date = li['updated']
-                l.vcs_url = li['vcs_url']
-                l.vcs_web_url = li['vcs_web_url']
-                l.vcs_web_tree_base_url = li['vcs_web_tree_base_url']
-                l.vcs_web_file_base_url = li['vcs_web_file_base_url']
                 l.summary = li['summary']
                 l.description = li['description']
+
+                if created:
+                    # predefined layers in the fixtures (for example poky.xml)
+                    # always preempt the Layer Index for these values
+                    l.vcs_url = li['vcs_url']
+                    l.vcs_web_url = li['vcs_web_url']
+                    l.vcs_web_tree_base_url = li['vcs_web_tree_base_url']
+                    l.vcs_web_file_base_url = li['vcs_web_file_base_url']
                 l.save()
             except Layer.MultipleObjectsReturned:
                 logger.info("Skipped %s as we found multiple layers and "
@@ -211,12 +188,14 @@
 
         total = len(layerbranches_info)
         for i, lbi in enumerate(layerbranches_info):
+            # release as defined by toaster map to layerindex branch
+            release = li_branch_id_to_toaster_release[lbi['branch']]
 
             try:
                 lv, created = Layer_Version.objects.get_or_create(
-                    layer_source=LayerSource.TYPE_LAYERINDEX,
                     layer=Layer.objects.get(
-                        pk=li_layer_id_to_toaster_layer_id[lbi['layer']])
+                        pk=li_layer_id_to_toaster_layer_id[lbi['layer']]),
+                    release=release
                 )
             except KeyError:
                 logger.warning(
@@ -224,11 +203,12 @@
                     lbi['layer'])
                 continue
 
-            lv.release = li_branch_id_to_toaster_release[lbi['branch']]
-            lv.up_date = lbi['updated']
-            lv.commit = lbi['actual_branch']
-            lv.dirpath = lbi['vcs_subdir']
-            lv.save()
+            if created:
+                lv.release = li_branch_id_to_toaster_release[lbi['branch']]
+                lv.up_date = lbi['updated']
+                lv.commit = lbi['actual_branch']
+                lv.dirpath = lbi['vcs_subdir']
+                lv.save()
 
             li_layer_branch_id_to_toaster_lv_id[lbi['id']] =\
                 lv.pk
@@ -255,9 +235,8 @@
                 layer_id = li_layer_id_to_toaster_layer_id[ldi['dependency']]
 
                 dependlist[lv].append(
-                    Layer_Version.objects.get(
-                        layer_source=LayerSource.TYPE_LAYERINDEX,
-                        layer__pk=layer_id))
+                    Layer_Version.objects.get(layer__pk=layer_id,
+                                              release=lv.release))
 
             except Layer_Version.DoesNotExist:
                 logger.warning("Cannot find layer version (ls:%s),"
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/orm/models.py b/import-layers/yocto-poky/bitbake/lib/toaster/orm/models.py
index a7de57c..a49f9a4 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/orm/models.py
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/orm/models.py
@@ -38,6 +38,7 @@
 import itertools
 from signal import SIGUSR1
 
+
 import logging
 logger = logging.getLogger("toaster")
 
@@ -178,24 +179,27 @@
         else:
             return projects[0]
 
+
 class Project(models.Model):
-    search_allowed_fields = ['name', 'short_description', 'release__name', 'release__branch_name']
+    search_allowed_fields = ['name', 'short_description', 'release__name',
+                             'release__branch_name']
     name = models.CharField(max_length=100)
     short_description = models.CharField(max_length=50, blank=True)
     bitbake_version = models.ForeignKey('BitbakeVersion', null=True)
-    release     = models.ForeignKey("Release", null=True)
-    created     = models.DateTimeField(auto_now_add = True)
-    updated     = models.DateTimeField(auto_now = True)
+    release = models.ForeignKey("Release", null=True)
+    created = models.DateTimeField(auto_now_add=True)
+    updated = models.DateTimeField(auto_now=True)
     # This is a horrible hack; since Toaster has no "User" model available when
     # running in interactive mode, we can't reference the field here directly
-    # Instead, we keep a possible null reference to the User id, as not to force
+    # Instead, we keep a possible null reference to the User id,
+    # as not to force
     # hard links to possibly missing models
-    user_id     = models.IntegerField(null = True)
-    objects     = ProjectManager()
+    user_id = models.IntegerField(null=True)
+    objects = ProjectManager()
 
     # set to True for the project which is the default container
     # for builds initiated by the command line etc.
-    is_default  = models.BooleanField(default = False)
+    is_default= models.BooleanField(default=False)
 
     def __unicode__(self):
         return "%s (Release %s, BBV %s)" % (self.name, self.release, self.bitbake_version)
@@ -221,16 +225,16 @@
             return( -1 )
 
     def get_last_outcome(self):
-        build_id = self.get_last_build_id
+        build_id = self.get_last_build_id()
         if (-1 == build_id):
             return( "" )
         try:
-            return Build.objects.filter( id = self.get_last_build_id )[ 0 ].outcome
+            return Build.objects.filter( id = build_id )[ 0 ].outcome
         except (Build.DoesNotExist,IndexError):
             return( "not_found" )
 
     def get_last_target(self):
-        build_id = self.get_last_build_id
+        build_id = self.get_last_build_id()
         if (-1 == build_id):
             return( "" )
         try:
@@ -239,7 +243,7 @@
             return( "not_found" )
 
     def get_last_errors(self):
-        build_id = self.get_last_build_id
+        build_id = self.get_last_build_id()
         if (-1 == build_id):
             return( 0 )
         try:
@@ -248,7 +252,7 @@
             return( "not_found" )
 
     def get_last_warnings(self):
-        build_id = self.get_last_build_id
+        build_id = self.get_last_build_id()
         if (-1 == build_id):
             return( 0 )
         try:
@@ -265,7 +269,7 @@
         return last_build.get_image_file_extensions()
 
     def get_last_imgfiles(self):
-        build_id = self.get_last_build_id
+        build_id = self.get_last_build_id()
         if (-1 == build_id):
             return( "" )
         try:
@@ -333,20 +337,45 @@
 
         return queryset
 
-
     def schedule_build(self):
-        from bldcontrol.models import BuildRequest, BRTarget, BRLayer, BRVariable, BRBitbake
-        br = BuildRequest.objects.create(project = self)
-        try:
 
-            BRBitbake.objects.create(req = br,
-                giturl = self.bitbake_version.giturl,
-                commit = self.bitbake_version.branch,
-                dirpath = self.bitbake_version.dirpath)
+        from bldcontrol.models import BuildRequest, BRTarget, BRLayer
+        from bldcontrol.models import BRBitbake, BRVariable
+
+        try:
+            now = timezone.now()
+            build = Build.objects.create(project=self,
+                                         completed_on=now,
+                                         started_on=now)
+
+            br = BuildRequest.objects.create(project=self,
+                                             state=BuildRequest.REQ_QUEUED,
+                                             build=build)
+            BRBitbake.objects.create(req=br,
+                                     giturl=self.bitbake_version.giturl,
+                                     commit=self.bitbake_version.branch,
+                                     dirpath=self.bitbake_version.dirpath)
+
+            for t in self.projecttarget_set.all():
+                BRTarget.objects.create(req=br, target=t.target, task=t.task)
+                Target.objects.create(build=br.build, target=t.target,
+                                      task=t.task)
+                # If we're about to build a custom image recipe make sure
+                # that layer is currently in the project before we create the
+                # BRLayer objects
+                customrecipe = CustomImageRecipe.objects.filter(
+                    name=t.target,
+                    project=self).first()
+                if customrecipe:
+                    ProjectLayer.objects.get_or_create(
+                        project=self,
+                        layercommit=customrecipe.layer_version,
+                        optional=False)
 
             for l in self.projectlayer_set.all().order_by("pk"):
                 commit = l.layercommit.get_vcs_reference()
-                print("ii Building layer ", l.layercommit.layer.name, " at vcs point ", commit)
+                logger.debug("Adding layer to build %s" %
+                             l.layercommit.layer.name)
                 BRLayer.objects.create(
                     req=br,
                     name=l.layercommit.layer.name,
@@ -357,25 +386,16 @@
                     local_source_dir=l.layercommit.layer.local_source_dir
                 )
 
-            br.state = BuildRequest.REQ_QUEUED
-            now = timezone.now()
-            br.build = Build.objects.create(project = self,
-                                completed_on=now,
-                                started_on=now,
-                                )
-            for t in self.projecttarget_set.all():
-                BRTarget.objects.create(req = br, target = t.target, task = t.task)
-                Target.objects.create(build = br.build, target = t.target, task = t.task)
-
             for v in self.projectvariable_set.all():
-                BRVariable.objects.create(req = br, name = v.name, value = v.value)
-
+                BRVariable.objects.create(req=br, name=v.name, value=v.value)
 
             try:
-                br.build.machine = self.projectvariable_set.get(name = 'MACHINE').value
+                br.build.machine = self.projectvariable_set.get(
+                    name='MACHINE').value
                 br.build.save()
             except ProjectVariable.DoesNotExist:
                 pass
+
             br.save()
             signal_runbuilds()
 
@@ -882,7 +902,7 @@
         'ext4.gz', 'ext3', 'ext3.gz', 'hdddirect', 'hddimg', 'iso', 'jffs2',
         'jffs2.sum', 'multiubi', 'qcow2', 'squashfs', 'squashfs-lzo',
         'squashfs-xz', 'tar', 'tar.bz2', 'tar.gz', 'tar.lz4', 'tar.xz', 'ubi',
-        'ubifs', 'vdi', 'vmdk', 'wic', 'wic.bz2', 'wic.gz', 'wic.lzma'
+        'ubifs', 'vdi', 'vmdk', 'wic', 'wic.bmap', 'wic.bz2', 'wic.gz', 'wic.lzma'
     }
 
     target = models.ForeignKey(Target)
@@ -1365,7 +1385,7 @@
     name = models.CharField(max_length=100)
     layer_index_url = models.URLField()
     vcs_url = GitURLField(default=None, null=True)
-    local_source_dir = models.TextField(null = True, default = None)
+    local_source_dir = models.TextField(null=True, default=None)
     vcs_web_url = models.URLField(null=True, default=None)
     vcs_web_tree_base_url = models.URLField(null=True, default=None)
     vcs_web_file_base_url = models.URLField(null=True, default=None)
@@ -1473,22 +1493,33 @@
             return self.commit
         return 'N/A'
 
-    def get_detailspage_url(self, project_id):
+    def get_detailspage_url(self, project_id=None):
+        """ returns the url to the layer details page uses own project
+        field if project_id is not specified """
+
+        if project_id is None:
+            project_id = self.project.pk
+
         return reverse('layerdetails', args=(project_id, self.pk))
 
     def get_alldeps(self, project_id):
         """Get full list of unique layer dependencies."""
-        def gen_layerdeps(lver, project):
+        def gen_layerdeps(lver, project, depth):
+            if depth == 0:
+                return
             for ldep in lver.dependencies.all():
                 yield ldep.depends_on
                 # get next level of deps recursively calling gen_layerdeps
-                for subdep in gen_layerdeps(ldep.depends_on, project):
+                for subdep in gen_layerdeps(ldep.depends_on, project, depth-1):
                     yield subdep
 
         project = Project.objects.get(pk=project_id)
         result = []
-        projectlvers = [player.layercommit for player in project.projectlayer_set.all()]
-        for dep in gen_layerdeps(self, project):
+        projectlvers = [player.layercommit for player in
+                        project.projectlayer_set.all()]
+        # protect against infinite layer dependency loops
+        maxdepth = 20
+        for dep in gen_layerdeps(self, project, maxdepth):
             # filter out duplicates and layers already belonging to the project
             if dep not in result + projectlvers:
                 result.append(dep)
@@ -1631,7 +1662,8 @@
         if base_recipe_path:
             base_recipe = open(base_recipe_path, 'r').read()
         else:
-            raise IOError("Based on recipe file not found")
+            raise IOError("Based on recipe file not found: %s" %
+                          base_recipe_path)
 
         # Add a special case for when the recipe we have based a custom image
         # recipe on requires another recipe.
@@ -1741,8 +1773,12 @@
 
 def signal_runbuilds():
     """Send SIGUSR1 to runbuilds process"""
-    with open(os.path.join(os.getenv('BUILDDIR'), '.runbuilds.pid')) as pidf:
-        os.kill(int(pidf.read()), SIGUSR1)
+    try:
+        with open(os.path.join(os.getenv('BUILDDIR', '.'),
+                               '.runbuilds.pid')) as pidf:
+            os.kill(int(pidf.read()), SIGUSR1)
+    except FileNotFoundError:
+        logger.info("Stopping existing runbuilds: no current process found")
 
 django.db.models.signals.post_save.connect(invalidate_cache)
 django.db.models.signals.post_delete.connect(invalidate_cache)
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/tests/browser/README b/import-layers/yocto-poky/bitbake/lib/toaster/tests/browser/README
index 6b09d20..352c4fe 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/tests/browser/README
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/tests/browser/README
@@ -2,9 +2,9 @@
 
 These tests require Selenium to be installed in your Python environment.
 
-The simplest way to install this is via pip:
+The simplest way to install this is via pip3:
 
-  pip install selenium==2.53.2
+  pip3 install selenium==2.53.2
 
 Note that if you use other versions of Selenium, some of the tests (such as
 tests.browser.test_js_unit_tests.TestJsUnitTests) may fail, as these rely on
@@ -18,7 +18,7 @@
 * On Windows, put chromedriver.exe in the same directory as chrome.exe
 
 To run tests against PhantomJS (headless):
-
+--NOTE - Selenium seems to be deprecating support for this mode ---
 * Download and install PhantomJS:
   http://phantomjs.org/download.html
 * On *nix systems, put phantomjs on PATH
@@ -43,13 +43,30 @@
 The test cases will instantiate a Selenium driver set by the
 TOASTER_TESTS_BROWSER environment variable, or Chrome if this is not specified.
 
+To run tests against the Selenium Firefox Docker container:
+More explanation is located at https://wiki.yoctoproject.org/wiki/TipsAndTricks/TestingToasterWithContainers
+* Run the Selenium container:
+  ** docker run -it --rm=true -p 5900:5900 -p 4444:4444 --name=selenium selenium/standalone-firefox-debug:2.53.0
+  *** 5900 is the default vnc port. If you are runing a vnc server on your machine map a different port e.g. -p 6900:5900 and connect vnc client to 127.0.0.1:6900
+  *** 4444 is the default selenium sever port.
+* Run the tests
+  ** TOASTER_TESTS_BROWSER=http://127.0.0.1:4444/wd/hub TOASTER_TESTS_URL=http://172.17.0.1:8000 ./bitbake/lib/toaster/manage.py test --liveserver=172.17.0.1:8000 tests.browser
+  ** TOASTER_TESTS_BROWSER=remote TOASTER_REMOTE_HUB=http://127.0.0.1:4444/wd/hub ./bitbake/lib/toaster/manage.py test --liveserver=172.17.0.1:8000 tests.browser
+  *** TOASTER_REMOTE_HUB - This is the address for the Selenium Remote Web Driver hub. Assuming you ran the contianer with -p 4444:4444 it will be http://127.0.0.1:4444/wd/hub.
+  ***  --liveserver=xxx tells Django to run the test server on an interface and port reachable by both host and container.
+  **** 172.17.0.1 is the default docker bridge on linux, viewable from inside and outside the contianers. Find it with "ip -4 addr show dev docker0"
+* connect to the vnc server to see the tests if you would like
+   ** xtightvncviewer 127.0.0.1:5900
+   ** note, you need to wait for the test container to come up before this can connect.
+
 Available drivers:
 
 * chrome (default)
 * firefox
 * marionette (for newer Firefoxes)
 * ie
-* phantomjs
+* phantomjs (deprecated)
+* remote
 
 e.g. to run the test suite with phantomjs where you have phantomjs installed
 in /home/me/apps/phantomjs:
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/tests/browser/selenium_helpers_base.py b/import-layers/yocto-poky/bitbake/lib/toaster/tests/browser/selenium_helpers_base.py
index 14e9c15..156d639 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/tests/browser/selenium_helpers_base.py
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/tests/browser/selenium_helpers_base.py
@@ -39,7 +39,7 @@
 from selenium.common.exceptions import NoSuchElementException, \
         StaleElementReferenceException, TimeoutException
 
-def create_selenium_driver(browser='chrome'):
+def create_selenium_driver(cls,browser='chrome'):
     # set default browser string based on env (if available)
     env_browser = os.environ.get('TOASTER_TESTS_BROWSER')
     if env_browser:
@@ -59,6 +59,15 @@
         return webdriver.Ie()
     elif browser == 'phantomjs':
         return webdriver.PhantomJS()
+    elif browser == 'remote':
+        # if we were to add yet another env variable like TOASTER_REMOTE_BROWSER
+        # we could let people pick firefox or chrome, left for later
+        remote_hub= os.environ.get('TOASTER_REMOTE_HUB')
+        driver = webdriver.Remote(remote_hub,
+                                  webdriver.DesiredCapabilities.FIREFOX.copy())
+
+        driver.get("http://%s:%s"%(cls.server_thread.host,cls.server_thread.port))
+        return driver
     else:
         msg = 'Selenium driver for browser %s is not available' % browser
         raise RuntimeError(msg)
@@ -135,7 +144,7 @@
 
         # instantiate the Selenium webdriver once for all the test methods
         # in this test case
-        cls.driver = create_selenium_driver()
+        cls.driver = create_selenium_driver(cls)
         cls.driver.maximize_window()
 
     @classmethod
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/tests/browser/test_layerdetails_page.py b/import-layers/yocto-poky/bitbake/lib/toaster/tests/browser/test_layerdetails_page.py
index 6392d1e..f24fb09 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/tests/browser/test_layerdetails_page.py
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/tests/browser/test_layerdetails_page.py
@@ -91,9 +91,10 @@
         for btn in self.find_all("dd .glyphicon-edit"):
             btn.click()
 
-        # Wait for the inputs to become visible
+        # Wait for the inputs to become visible after animation
         self.wait_until_visible("#layer-git input[type=text]")
         self.wait_until_visible("dd textarea")
+        self.wait_until_visible("dd .change-btn")
 
         # Edit each value
         for inputs in self.find_all("#layer-git input[type=text]") + \
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/tests/builds/buildtest.py b/import-layers/yocto-poky/bitbake/lib/toaster/tests/builds/buildtest.py
index fc7bd5b..5a56a11 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/tests/builds/buildtest.py
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/tests/builds/buildtest.py
@@ -24,82 +24,115 @@
 import time
 import unittest
 
-from orm.models import Project, Release, ProjectTarget, Build
+from orm.models import Project, Release, ProjectTarget, Build, ProjectVariable
 from bldcontrol.models import BuildEnvironment
 
-from bldcontrol.management.commands.loadconf import Command\
-    as LoadConfigCommand
-
 from bldcontrol.management.commands.runbuilds import Command\
     as RunBuildsCommand
 
+from django.core.management import call_command
+
 import subprocess
+import logging
+
+logger = logging.getLogger("toaster")
 
 # We use unittest.TestCase instead of django.test.TestCase because we don't
 # want to wrap everything in a database transaction as an external process
 # (bitbake needs access to the database)
 
+def load_build_environment():
+    call_command('loaddata', 'settings.xml', app_label="orm")
+    call_command('loaddata', 'poky.xml', app_label="orm")
+
+    current_builddir = os.environ.get("BUILDDIR")
+    if current_builddir:
+        BuildTest.BUILDDIR = current_builddir
+    else:
+        # Setup a builddir based on default layout
+        # bitbake inside openebedded-core
+        oe_init_build_env_path = os.path.join(
+            os.path.dirname(os.path.abspath(__file__)),
+            os.pardir,
+            os.pardir,
+            os.pardir,
+            os.pardir,
+            os.pardir,
+            'oe-init-build-env'
+        )
+        if not os.path.exists(oe_init_build_env_path):
+            raise Exception("We had no BUILDDIR set and couldn't "
+                            "find oe-init-build-env to set this up "
+                            "ourselves please run oe-init-build-env "
+                            "before running these tests")
+
+        oe_init_build_env_path = os.path.realpath(oe_init_build_env_path)
+        cmd = "bash -c 'source oe-init-build-env %s'" % BuildTest.BUILDDIR
+        p = subprocess.Popen(
+            cmd,
+            cwd=os.path.dirname(oe_init_build_env_path),
+            shell=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE)
+
+        output, err = p.communicate()
+        p.wait()
+
+        logger.info("oe-init-build-env %s %s" % (output, err))
+
+        os.environ['BUILDDIR'] = BuildTest.BUILDDIR
+
+    # Setup the path to bitbake we know where to find this
+    bitbake_path = os.path.join(
+        os.path.dirname(os.path.abspath(__file__)),
+        os.pardir,
+        os.pardir,
+        os.pardir,
+        os.pardir,
+        'bin',
+        'bitbake')
+    if not os.path.exists(bitbake_path):
+        raise Exception("Could not find bitbake at the expected path %s"
+                        % bitbake_path)
+
+    os.environ['BBBASEDIR'] = bitbake_path
 
 class BuildTest(unittest.TestCase):
 
     PROJECT_NAME = "Testbuild"
+    BUILDDIR = "/tmp/build/"
 
     def build(self, target):
         # So that the buildinfo helper uses the test database'
         self.assertEqual(
             os.environ.get('DJANGO_SETTINGS_MODULE', ''),
-            'toastermain.settings-test',
+            'toastermain.settings_test',
             "Please initialise django with the tests settings:  "
-            "DJANGO_SETTINGS_MODULE='toastermain.settings-test'")
+            "DJANGO_SETTINGS_MODULE='toastermain.settings_test'")
 
-        if self.target_already_built(target):
-            return
+        built = self.target_already_built(target)
+        if built:
+            return built
 
-        # Take a guess at the location of the toasterconf
-        poky_toaster_conf = '../../../meta-poky/conf/toasterconf.json'
-        oe_toaster_conf = '../../../meta/conf/toasterconf.json'
-        env_toaster_conf = os.environ.get('TOASTER_CONF')
-
-        config_file = None
-        if env_toaster_conf:
-            config_file = env_toaster_conf
-        else:
-            if os.path.exists(poky_toaster_conf):
-                config_file = poky_toaster_conf
-            elif os.path.exists(oe_toaster_conf):
-                config_file = oe_toaster_conf
-
-        self.assertIsNotNone(config_file,
-                             "Default locations for toasterconf not found"
-                             "please set $TOASTER_CONF manually")
-
-        # Setup the release information and default layers
-        print("\nImporting file: %s" % config_file)
-        os.environ['TOASTER_CONF'] = config_file
-        LoadConfigCommand()._import_layer_config(config_file)
-
-        os.environ['TOASTER_DIR'] = \
-            os.path.abspath(os.environ['BUILDDIR'] + "/../")
-
-        os.environ['BBBASEDIR'] = \
-            subprocess.check_output('which bitbake', shell=True)
+        load_build_environment()
 
         BuildEnvironment.objects.get_or_create(
             betype=BuildEnvironment.TYPE_LOCAL,
-            sourcedir=os.environ['TOASTER_DIR'],
-            builddir=os.environ['BUILDDIR']
+            sourcedir=BuildTest.BUILDDIR,
+            builddir=BuildTest.BUILDDIR
         )
 
         release = Release.objects.get(name='local')
 
         # Create a project for this build to run in
-        try:
-            project = Project.objects.get(name=BuildTest.PROJECT_NAME)
-        except Project.DoesNotExist:
-            project = Project.objects.create_project(
-                name=BuildTest.PROJECT_NAME,
-                release=release
-            )
+        project = Project.objects.create_project(name=BuildTest.PROJECT_NAME,
+                                                 release=release)
+
+        if os.environ.get("TOASTER_TEST_USE_SSTATE_MIRROR"):
+            ProjectVariable.objects.get_or_create(
+                name="SSTATE_MIRRORS",
+                value="file://.* http://autobuilder.yoctoproject.org/pub/sstate/PATH;downloadfilename=PATH",
+                project=project)
 
         ProjectTarget.objects.create(project=project,
                                      target=target,
@@ -118,9 +151,11 @@
             sys.stdout.flush()
             time.sleep(1)
 
-        self.assertNotEqual(build_request.build.outcome,
-                            Build.SUCCEEDED, "Build did not SUCCEEDED")
-        print("\nBuild finished")
+        self.assertEqual(Build.objects.get(pk=build_pk).outcome,
+                         Build.SUCCEEDED,
+                         "Build did not SUCCEEDED")
+
+        logger.info("\nBuild finished %s" % build_request.build.outcome)
         return build_request.build
 
     def target_already_built(self, target):
@@ -129,6 +164,6 @@
                 project__name=BuildTest.PROJECT_NAME):
             targets = build.target_set.values_list('target', flat=True)
             if target in targets:
-                return True
+                return build
 
-        return False
+        return None
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/tests/builds/test_core_image_min.py b/import-layers/yocto-poky/bitbake/lib/toaster/tests/builds/test_core_image_min.py
index dec0bfa..586f4a8 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/tests/builds/test_core_image_min.py
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/tests/builds/test_core_image_min.py
@@ -31,14 +31,14 @@
 from orm.models import Task_Dependency, Package, Target, Recipe
 from orm.models import CustomImagePackage
 
-from buildtest import BuildTest
+from tests.builds.buildtest import BuildTest
 
 
 class BuildCoreImageMinimal(BuildTest):
     """Build core-image-minimal and test the results"""
 
     def setUp(self):
-        self.build("core-image-minimal")
+        self.completed_build = self.build("core-image-minimal")
 
     # Check if build name is unique - tc_id=795
     def test_Build_Unique_Name(self):
@@ -59,38 +59,29 @@
 
     # Check if task order is unique for one build - tc=824
     def test_Task_Unique_Order(self):
-        builds = Build.objects.values('id')
-        cnt_err = []
+        total_task_order = Task.objects.filter(
+            build=self.built).values('order').count()
+        distinct_task_order = Task.objects.filter(
+            build=self.completed_build).values('order').distinct().count()
 
-        for build in builds:
-            total_task_order = Task.objects.filter(
-                build=build['id']).values('order').count()
-            distinct_task_order = Task.objects.filter(
-                build=build['id']).values('order').distinct().count()
-
-            if (total_task_order != distinct_task_order):
-                cnt_err.append(build['id'])
-
-        self.assertEqual(len(cnt_err),
-                         0,
-                         msg='Errors for build id: %s' % cnt_err)
+        self.assertEqual(total_task_order,
+                         distinct_task_order,
+                         msg='Errors task order is not unique')
 
     # Check task order sequence for one build - tc=825
     def test_Task_Order_Sequence(self):
-        builds = builds = Build.objects.values('id')
         cnt_err = []
-        for build in builds:
-            tasks = Task.objects.filter(
-                Q(build=build['id']),
-                ~Q(order=None),
-                ~Q(task_name__contains='_setscene')
-            ).values('id', 'order').order_by("order")
+        tasks = Task.objects.filter(
+            Q(build=self.completed_build),
+            ~Q(order=None),
+            ~Q(task_name__contains='_setscene')
+        ).values('id', 'order').order_by("order")
 
-            cnt_tasks = 0
-            for task in tasks:
-                cnt_tasks += 1
-                if (task['order'] != cnt_tasks):
-                    cnt_err.append(task['id'])
+        cnt_tasks = 0
+        for task in tasks:
+            cnt_tasks += 1
+            if (task['order'] != cnt_tasks):
+                cnt_err.append(task['id'])
         self.assertEqual(
             len(cnt_err), 0, msg='Errors for task id: %s' % cnt_err)
 
@@ -126,8 +117,7 @@
                     task['sstate_result'] != Task.SSTATE_MISS):
                 cnt_err.append({'id': task['id'],
                                 'name': task['task_name'],
-                                'sstate_result': task['sstate_result'],
-                               })
+                                'sstate_result': task['sstate_result']})
 
         self.assertEqual(len(cnt_err),
                          0,
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/log/.create b/import-layers/yocto-poky/bitbake/lib/toaster/tests/commands/__init__.py
similarity index 100%
rename from import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/log/.create
rename to import-layers/yocto-poky/bitbake/lib/toaster/tests/commands/__init__.py
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/tests/commands/test_loaddata.py b/import-layers/yocto-poky/bitbake/lib/toaster/tests/commands/test_loaddata.py
new file mode 100644
index 0000000..951f6ff
--- /dev/null
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/tests/commands/test_loaddata.py
@@ -0,0 +1,61 @@
+#! /usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+from django.test import TestCase
+from django.core import management
+
+from orm.models import Layer_Version, Layer, Release, ToasterSetting
+
+
+class TestLoadDataFixtures(TestCase):
+    """ Test loading our 3 provided fixtures """
+    def test_run_loaddata_poky_command(self):
+        management.call_command('loaddata', 'poky')
+
+        num_releases = Release.objects.count()
+
+        self.assertTrue(
+            Layer_Version.objects.filter(
+                layer__name="meta-poky").count() == num_releases,
+            "Loaded poky fixture but don't have a meta-poky for all releases"
+            " defined")
+
+    def test_run_loaddata_oecore_command(self):
+        management.call_command('loaddata', 'oe-core')
+
+        # We only have the one layer for oe-core setup
+        self.assertTrue(
+            Layer.objects.filter(name="openembedded-core").count() > 0,
+            "Loaded oe-core fixture but still have no openemebedded-core"
+            " layer")
+
+    def test_run_loaddata_settings_command(self):
+        management.call_command('loaddata', 'settings')
+
+        self.assertTrue(
+            ToasterSetting.objects.filter(name="DEFAULT_RELEASE").count() > 0,
+            "Loaded settings but have no DEFAULT_RELEASE")
+
+        self.assertTrue(
+            ToasterSetting.objects.filter(
+                name__startswith="DEFCONF").count() > 0,
+            "Loaded settings but have no DEFCONF (default project "
+            "configuration values)")
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/tests/commands/test_lsupdates.py b/import-layers/yocto-poky/bitbake/lib/toaster/tests/commands/test_lsupdates.py
new file mode 100644
index 0000000..49897a4
--- /dev/null
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/tests/commands/test_lsupdates.py
@@ -0,0 +1,45 @@
+#! /usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+from django.test import TestCase
+from django.core import management
+
+from orm.models import Layer_Version, Machine, Recipe
+
+
+class TestLayerIndexUpdater(TestCase):
+    def test_run_lsupdates_command(self):
+        # Load some release information for us to fetch from the layer index
+        management.call_command('loaddata', 'poky')
+
+        old_layers_count = Layer_Version.objects.count()
+        old_recipes_count = Recipe.objects.count()
+        old_machines_count = Machine.objects.count()
+
+        # Now fetch the metadata from the layer index
+        management.call_command('lsupdates')
+
+        self.assertTrue(Layer_Version.objects.count() > old_layers_count,
+                        "lsupdates ran but we still have no more layers!")
+        self.assertTrue(Recipe.objects.count() > old_recipes_count,
+                        "lsupdates ran but we still have no more Recipes!")
+        self.assertTrue(Machine.objects.count() > old_machines_count,
+                        "lsupdates ran but we still have no more Machines!")
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/tests/commands/test_runbuilds.py b/import-layers/yocto-poky/bitbake/lib/toaster/tests/commands/test_runbuilds.py
new file mode 100644
index 0000000..3e63483
--- /dev/null
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/tests/commands/test_runbuilds.py
@@ -0,0 +1,88 @@
+#! /usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import os
+
+from django.test import TestCase
+from django.core import management
+
+from orm.models import signal_runbuilds
+
+import threading
+import time
+import subprocess
+import signal
+
+
+class KillRunbuilds(threading.Thread):
+    """ Kill the runbuilds process after an amount of time """
+    def __init__(self, *args, **kwargs):
+        super(KillRunbuilds, self).__init__(*args, **kwargs)
+        self.setDaemon(True)
+
+    def run(self):
+        time.sleep(5)
+        signal_runbuilds()
+        time.sleep(1)
+
+        pidfile_path = os.path.join(os.environ.get("BUILDDIR", "."),
+                                    ".runbuilds.pid")
+
+        with open(pidfile_path) as pidfile:
+            pid = pidfile.read()
+            os.kill(int(pid), signal.SIGTERM)
+
+
+class TestCommands(TestCase):
+    """ Sanity test that runbuilds executes OK """
+
+    def setUp(self):
+        os.environ.setdefault("DJANGO_SETTINGS_MODULE",
+                              "toastermain.settings_test")
+        os.environ.setdefault("BUILDDIR",
+                              "/tmp/")
+
+        # Setup a real database if needed for runbuilds process
+        # to connect to
+        management.call_command('migrate')
+
+    def test_runbuilds_command(self):
+        kill_runbuilds = KillRunbuilds()
+        kill_runbuilds.start()
+
+        manage_py = os.path.join(
+            os.path.dirname(os.path.abspath(__file__)),
+            os.pardir,
+            os.pardir,
+            "manage.py")
+
+        command = "%s runbuilds" % manage_py
+
+        process = subprocess.Popen(command,
+                                   shell=True,
+                                   stdout=subprocess.PIPE,
+                                   stderr=subprocess.PIPE)
+
+        (out, err) = process.communicate()
+        process.wait()
+
+        self.assertNotEqual(process.returncode, 1,
+                            "Runbuilds returned an error %s" % err)
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/tests/eventreplay/README b/import-layers/yocto-poky/bitbake/lib/toaster/tests/eventreplay/README
new file mode 100644
index 0000000..8c5bb64
--- /dev/null
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/tests/eventreplay/README
@@ -0,0 +1,22 @@
+# Running eventreplay tests
+
+These tests use event log files produced by bitbake <target> -w <event log file>
+You need to have event log files produced before running this tests.
+
+At the moment of writing this document tests use 2 event log files: zlib.events
+and core-image-minimal.events. They're not provided with the tests due to their
+significant size.
+
+Here is how to produce them:
+
+$ . oe-init-build-env
+$ rm -r tmp sstate-cache
+$ bitbake core-image-minimal -w core-image-minimal.events
+$ rm -rf tmp sstate-cache
+$ bitbake zlib -w zlib.events
+
+After that it should be possible to run eventreplay tests this way:
+
+$ EVENTREPLAY_DIR=./ DJANGO_SETTINGS_MODULE=toastermain.settings_test ../bitbake/lib/toaster/manage.py test -v2 tests.eventreplay
+
+Note that environment variable EVENTREPLAY_DIR should point to the directory with event log files.
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/tests/eventreplay/__init__.py b/import-layers/yocto-poky/bitbake/lib/toaster/tests/eventreplay/__init__.py
new file mode 100644
index 0000000..6956619
--- /dev/null
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/tests/eventreplay/__init__.py
@@ -0,0 +1,97 @@
+#! /usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+# Tests were part of openembedded-core oe selftest Authored by: Lucian Musat
+# Ionut Chisanovici, Paul Eggleton and Cristian Iorga
+
+"""
+Test toaster backend by playing build event log files
+using toaster-eventreplay script
+"""
+
+import os
+
+from subprocess import getstatusoutput
+from pathlib import Path
+
+from django.test import TestCase
+
+from orm.models import Target_Installed_Package, Package, Build
+
+class EventReplay(TestCase):
+    """Base class for eventreplay test cases"""
+
+    def setUp(self):
+        """
+        Setup build environment:
+            - set self.script to toaster-eventreplay path
+            - set self.eventplay_dir to the value of EVENTPLAY_DIR env variable
+        """
+        bitbake_dir = Path(__file__.split('lib/toaster')[0])
+        self.script = bitbake_dir /  'bin' / 'toaster-eventreplay'
+        self.assertTrue(self.script.exists(), "%s doesn't exist")
+        self.eventplay_dir = os.getenv("EVENTREPLAY_DIR")
+        self.assertTrue(self.eventplay_dir,
+                        "Environment variable EVENTREPLAY_DIR is not set")
+
+    def _replay(self, eventfile):
+        """Run toaster-eventplay <eventfile>"""
+        eventpath = Path(self.eventplay_dir) / eventfile
+        status, output = getstatusoutput('%s %s' % (self.script, eventpath))
+        if status:
+            print(output)
+
+        self.assertEqual(status, 0)
+
+class CoreImageMinimalEventReplay(EventReplay):
+    """Replay core-image-minimal events"""
+
+    def test_installed_packages(self):
+        """Test if all required packages have been installed"""
+
+        self._replay('core-image-minimal.events')
+
+        # test installed packages
+        packages = sorted(Target_Installed_Package.objects.\
+                          values_list('package__name', flat=True))
+        self.assertEqual(packages, ['base-files', 'base-passwd', 'busybox',
+                                    'busybox-hwclock', 'busybox-syslog',
+                                    'busybox-udhcpc', 'eudev', 'glibc',
+                                    'init-ifupdown', 'initscripts',
+                                    'initscripts-functions', 'kernel-base',
+                                    'kernel-module-uvesafb', 'libkmod',
+                                    'modutils-initscripts', 'netbase',
+                                    'packagegroup-core-boot', 'run-postinsts',
+                                    'sysvinit', 'sysvinit-inittab',
+                                    'sysvinit-pidof', 'udev-cache',
+                                    'update-alternatives-opkg',
+                                    'update-rc.d', 'util-linux-libblkid',
+                                    'util-linux-libuuid', 'v86d', 'zlib'])
+
+class ZlibEventReplay(EventReplay):
+    """Replay zlib events"""
+
+    def test_replay_zlib(self):
+        """Test if zlib build and package are in the database"""
+        self._replay("zlib.events")
+
+        self.assertEqual(Build.objects.last().target_set.last().target, "zlib")
+        self.assertTrue('zlib' in Package.objects.values_list('name', flat=True))
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/log/.create b/import-layers/yocto-poky/bitbake/lib/toaster/tests/functional/__init__.py
similarity index 100%
copy from import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/log/.create
copy to import-layers/yocto-poky/bitbake/lib/toaster/tests/functional/__init__.py
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/tests/functional/functional_helpers.py b/import-layers/yocto-poky/bitbake/lib/toaster/tests/functional/functional_helpers.py
new file mode 100644
index 0000000..486078a
--- /dev/null
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/tests/functional/functional_helpers.py
@@ -0,0 +1,122 @@
+#! /usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# BitBake Toaster functional tests implementation
+#
+# Copyright (C) 2017 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import os
+import logging
+import subprocess
+import signal
+import time
+import re
+
+from tests.browser.selenium_helpers_base import SeleniumTestCaseBase
+from tests.builds.buildtest import load_build_environment
+
+logger = logging.getLogger("toaster")
+
+class SeleniumFunctionalTestCase(SeleniumTestCaseBase):
+    wait_toaster_time = 5
+
+    @classmethod
+    def setUpClass(cls):
+        # So that the buildinfo helper uses the test database'
+        if os.environ.get('DJANGO_SETTINGS_MODULE', '') != \
+            'toastermain.settings_test':
+            raise RuntimeError("Please initialise django with the tests settings:  " \
+                "DJANGO_SETTINGS_MODULE='toastermain.settings_test'")
+
+        load_build_environment()
+
+        # start toaster
+        cmd = "bash -c 'source toaster start'"
+        p = subprocess.Popen(
+            cmd,
+            cwd=os.environ.get("BUILDDIR"),
+            shell=True)
+        if p.wait() != 0:
+            raise RuntimeError("Can't initialize toaster")
+
+        super(SeleniumFunctionalTestCase, cls).setUpClass()
+        cls.live_server_url = 'http://localhost:8000/'
+
+    @classmethod
+    def tearDownClass(cls):
+        super(SeleniumFunctionalTestCase, cls).tearDownClass()
+
+        # XXX: source toaster stop gets blocked, to review why?
+        # from now send SIGTERM by hand
+        time.sleep(cls.wait_toaster_time)
+        builddir = os.environ.get("BUILDDIR")
+
+        with open(os.path.join(builddir, '.toastermain.pid'), 'r') as f:
+            toastermain_pid = int(f.read())
+            os.kill(toastermain_pid, signal.SIGTERM)
+        with open(os.path.join(builddir, '.runbuilds.pid'), 'r') as f:
+            runbuilds_pid = int(f.read())
+            os.kill(runbuilds_pid, signal.SIGTERM)
+
+
+    def get_URL(self):
+         rc=self.get_page_source()
+         project_url=re.search("(projectPageUrl\s:\s\")(.*)(\",)",rc)
+         return project_url.group(2)
+
+
+    def find_element_by_link_text_in_table(self, table_id, link_text):
+        """
+        Assume there're multiple suitable "find_element_by_link_text".
+        In this circumstance we need to specify "table".
+        """
+        try:
+            table_element = self.get_table_element(table_id)
+            element = table_element.find_element_by_link_text(link_text)
+        except NoSuchElementException as e:
+            print('no element found')
+            raise
+        return element
+
+    def get_table_element(self, table_id, *coordinate):
+        if len(coordinate) == 0:
+#return whole-table element
+            element_xpath = "//*[@id='" + table_id + "']"
+            try:
+                element = self.driver.find_element_by_xpath(element_xpath)
+            except NoSuchElementException as e:
+                raise
+            return element
+        row = coordinate[0]
+
+        if len(coordinate) == 1:
+#return whole-row element
+            element_xpath = "//*[@id='" + table_id + "']/tbody/tr[" + str(row) + "]"
+            try:
+                element = self.driver.find_element_by_xpath(element_xpath)
+            except NoSuchElementException as e:
+                return False
+            return element
+#now we are looking for an element with specified X and Y
+        column = coordinate[1]
+
+        element_xpath = "//*[@id='" + table_id + "']/tbody/tr[" + str(row) + "]/td[" + str(column) + "]"
+        try:
+            element = self.driver.find_element_by_xpath(element_xpath)
+        except NoSuchElementException as e:
+            return False
+        return element
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/tests/functional/test_functional_basic.py b/import-layers/yocto-poky/bitbake/lib/toaster/tests/functional/test_functional_basic.py
new file mode 100644
index 0000000..cfa2b0f
--- /dev/null
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/tests/functional/test_functional_basic.py
@@ -0,0 +1,243 @@
+#! /usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# BitBake Toaster functional tests implementation
+#
+# Copyright (C) 2017 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import time
+import re
+from tests.functional.functional_helpers import SeleniumFunctionalTestCase
+from orm.models import Project
+
+class FuntionalTestBasic(SeleniumFunctionalTestCase):
+
+#   testcase (1514)
+    def test_create_slenium_project(self):
+        project_name = 'selenium-project'
+        self.get('')
+        self.driver.find_element_by_link_text("To start building, create your first Toaster project").click()
+        self.driver.find_element_by_id("new-project-name").send_keys(project_name)
+        self.driver.find_element_by_id('projectversion').click()
+        self.driver.find_element_by_id("create-project-button").click()
+        element = self.wait_until_visible('#project-created-notification')
+        self.assertTrue(self.element_exists('#project-created-notification'),'Project creation notification not shown')
+        self.assertTrue(project_name in element.text,
+                        "New project name not in new project notification")
+        self.assertTrue(Project.objects.filter(name=project_name).count(),
+                        "New project not found in database")
+
+ #  testcase (1515)
+    def test_verify_left_bar_menu(self):
+        self.get('')
+        self.wait_until_visible('#projectstable')
+        self.find_element_by_link_text_in_table('projectstable', 'selenium-project').click()
+        self.assertTrue(self.element_exists('#config-nav'),'Configuration Tab does not exist')
+        project_URL=self.get_URL()
+        self.driver.find_element_by_xpath('//a[@href="'+project_URL+'"]').click()
+
+        try:
+            self.driver.find_element_by_xpath("//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'customimages/"'+"]").click()
+            self.assertTrue(re.search("Custom images",self.driver.find_element_by_xpath("//div[@class='col-md-10']").text),'Custom images information is not loading properly')
+        except:
+            self.fail(msg='No Custom images tab available')
+
+        try:
+            self.driver.find_element_by_xpath("//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'images/"'+"]").click()
+            self.assertTrue(re.search("Compatible image recipes",self.driver.find_element_by_xpath("//div[@class='col-md-10']").text),'The Compatible image recipes information is not loading properly')
+        except:
+            self.fail(msg='No Compatible image tab available')
+
+        try:
+            self.driver.find_element_by_xpath("//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'softwarerecipes/"'+"]").click()
+            self.assertTrue(re.search("Compatible software recipes",self.driver.find_element_by_xpath("//div[@class='col-md-10']").text),'The Compatible software recipe information is not loading properly')
+        except:
+            self.fail(msg='No Compatible software recipe tab available')
+
+        try:
+            self.driver.find_element_by_xpath("//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'machines/"'+"]").click()
+            self.assertTrue(re.search("Compatible machines",self.driver.find_element_by_xpath("//div[@class='col-md-10']").text),'The Compatible machine information is not loading properly')
+        except:
+            self.fail(msg='No Compatible machines tab available')
+
+        try:
+            self.driver.find_element_by_xpath("//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'layers/"'+"]").click()
+            self.assertTrue(re.search("Compatible layers",self.driver.find_element_by_xpath("//div[@class='col-md-10']").text),'The Compatible layer information is not loading properly')
+        except:
+            self.fail(msg='No Compatible layers tab available')
+
+        try:
+            self.driver.find_element_by_xpath("//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'configuration"'+"]").click()
+            self.assertTrue(re.search("Bitbake variables",self.driver.find_element_by_xpath("//div[@class='col-md-10']").text),'The Bitbake variables information is not loading properly')
+        except:
+            self.fail(msg='No Bitbake variables tab available')
+
+#   testcase (1516)
+    def test_review_configuration_information(self):
+        self.get('')
+        self.driver.find_element_by_xpath("//div[@id='global-nav']/ul/li/a[@href="+'"'+'/toastergui/projects/'+'"'+"]").click()
+        self.wait_until_visible('#projectstable')
+        self.find_element_by_link_text_in_table('projectstable', 'selenium-project').click()
+        project_URL=self.get_URL()
+
+        try:
+           self.assertTrue(self.element_exists('#machine-section'),'Machine section for the project configuration page does not exist')
+           self.assertTrue(re.search("qemux86",self.driver.find_element_by_xpath("//span[@id='project-machine-name']").text),'The machine type is not assigned')
+           self.driver.find_element_by_xpath("//span[@id='change-machine-toggle']").click()
+           self.wait_until_visible('#select-machine-form')
+           self.wait_until_visible('#cancel-machine-change')
+           self.driver.find_element_by_xpath("//form[@id='select-machine-form']/a[@id='cancel-machine-change']").click()
+        except:
+           self.fail(msg='The machine information is wrong in the configuration page')
+
+        try:
+           self.driver.find_element_by_id('no-most-built')
+        except:
+           self.fail(msg='No Most built information in project detail page')
+
+        try:
+           self.assertTrue(re.search("Yocto Project master",self.driver.find_element_by_xpath("//span[@id='project-release-title']").text),'The project release is not defined')
+        except:
+           self.fail(msg='No project release title information in project detail page')
+
+        try:
+           self.driver.find_element_by_xpath("//div[@id='layer-container']")
+           self.assertTrue(re.search("3",self.driver.find_element_by_id("project-layers-count").text),'There should be 3 layers listed in the layer count')
+           layer_list = self.driver.find_element_by_id("layers-in-project-list")
+           layers = layer_list.find_elements_by_tag_name("li")
+           for layer in layers:
+               if re.match ("openembedded-core",layer.text):
+                    print ("openembedded-core layer is a default layer in the project configuration")
+               elif re.match ("meta-poky",layer.text):
+                  print ("meta-poky layer is a default layer in the project configuration")
+               elif re.match ("meta-yocto-bsp",layer.text):
+                  print ("meta-yocto-bsp is a default layer in the project configuratoin")
+               else:
+                  self.fail(msg='default layers are missing from the project configuration')
+        except:
+           self.fail(msg='No Layer information in project detail page')
+
+#   testcase (1517)
+    def test_verify_machine_information(self):
+        self.get('')
+        self.driver.find_element_by_xpath("//div[@id='global-nav']/ul/li/a[@href="+'"'+'/toastergui/projects/'+'"'+"]").click()
+        self.wait_until_visible('#projectstable')
+        self.find_element_by_link_text_in_table('projectstable', 'selenium-project').click()
+
+        try:
+            self.assertTrue(self.element_exists('#machine-section'),'Machine section for the project configuration page does not exist')
+            self.assertTrue(re.search("qemux86",self.driver.find_element_by_id("project-machine-name").text),'The machine type is not assigned')
+            self.driver.find_element_by_id("change-machine-toggle").click()
+            self.wait_until_visible('#select-machine-form')
+            self.wait_until_visible('#cancel-machine-change')
+            self.driver.find_element_by_id("cancel-machine-change").click()
+        except:
+            self.fail(msg='The machine information is wrong in the configuration page')
+
+#   testcase (1518)
+    def test_verify_most_built_recipes_information(self):
+        self.get('')
+        self.driver.find_element_by_xpath("//div[@id='global-nav']/ul/li/a[@href="+'"'+'/toastergui/projects/'+'"'+"]").click()
+        self.wait_until_visible('#projectstable')
+        self.find_element_by_link_text_in_table('projectstable', 'selenium-project').click()
+        project_URL=self.get_URL()
+
+        try:
+            self.assertTrue(re.search("You haven't built any recipes yet",self.driver.find_element_by_id("no-most-built").text),'Default message of no builds is not present')
+            self.driver.find_element_by_xpath("//div[@id='no-most-built']/p/a[@href="+'"'+project_URL+'images/"'+"]").click()
+            self.assertTrue(re.search("Compatible image recipes",self.driver.find_element_by_xpath("//div[@class='col-md-10']").text),'The Choose a recipe to build link  is not working  properly')
+        except:
+            self.fail(msg='No Most built information in project detail page')
+
+#   testcase (1519)
+    def test_verify_project_release_information(self):
+        self.get('')
+        self.driver.find_element_by_xpath("//div[@id='global-nav']/ul/li/a[@href="+'"'+'/toastergui/projects/'+'"'+"]").click()
+        self.wait_until_visible('#projectstable')
+        self.find_element_by_link_text_in_table('projectstable', 'selenium-project').click()
+
+        try:
+            self.assertTrue(re.search("Yocto Project master",self.driver.find_element_by_id("project-release-title").text),'The project release is not defined')
+        except:
+            self.fail(msg='No project release title information in project detail page')
+
+#   testcase (1520)
+    def test_verify_layer_information(self):
+        self.get('')
+        self.driver.find_element_by_xpath("//div[@id='global-nav']/ul/li/a[@href="+'"'+'/toastergui/projects/'+'"'+"]").click()
+        self.wait_until_visible('#projectstable')
+        self.find_element_by_link_text_in_table('projectstable', 'selenium-project').click()
+        project_URL=self.get_URL()
+
+        try:
+           self.driver.find_element_by_xpath("//div[@id='layer-container']")
+           self.assertTrue(re.search("3",self.driver.find_element_by_id("project-layers-count").text),'There should be 3 layers listed in the layer count')
+           layer_list = self.driver.find_element_by_id("layers-in-project-list")
+           layers = layer_list.find_elements_by_tag_name("li")
+
+           for layer in layers:
+               if re.match ("openembedded-core",layer.text):
+                    print ("openembedded-core layer is a default layer in the project configuration")
+               elif re.match ("meta-poky",layer.text):
+                  print ("meta-poky layer is a default layer in the project configuration")
+               elif re.match ("meta-yocto-bsp",layer.text):
+                  print ("meta-yocto-bsp is a default layer in the project configuratoin")
+               else:
+                  self.fail(msg='default layers are missing from the project configuration')
+
+           self.driver.find_element_by_xpath("//input[@id='layer-add-input']")
+           self.driver.find_element_by_xpath("//button[@id='add-layer-btn']")
+           self.driver.find_element_by_xpath("//div[@id='layer-container']/form[@class='form-inline']/p/a[@id='view-compatible-layers']")
+           self.driver.find_element_by_xpath("//div[@id='layer-container']/form[@class='form-inline']/p/a[@href="+'"'+project_URL+'importlayer"'+"]")
+        except:
+            self.fail(msg='No Layer information in project detail page')
+
+#   testcase (1521)
+    def test_verify_project_detail_links(self):
+        self.get('')
+        self.driver.find_element_by_xpath("//div[@id='global-nav']/ul/li/a[@href="+'"'+'/toastergui/projects/'+'"'+"]").click()
+        self.wait_until_visible('#projectstable')
+        self.find_element_by_link_text_in_table('projectstable', 'selenium-project').click()
+        project_URL=self.get_URL()
+
+        self.driver.find_element_by_xpath("//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li[@id='topbar-configuration-tab']/a[@href="+'"'+project_URL+'"'+"]").click()
+        self.assertTrue(re.search("Configuration",self.driver.find_element_by_xpath("//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li[@id='topbar-configuration-tab']/a[@href="+'"'+project_URL+'"'+"]").text), 'Configuration tab in project topbar is misspelled')
+
+        try:
+            self.driver.find_element_by_xpath("//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'builds/"'+"]").click()
+            self.assertTrue(re.search("Builds",self.driver.find_element_by_xpath("//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'builds/"'+"]").text), 'Builds tab in project topbar is misspelled')
+            self.driver.find_element_by_xpath("//div[@id='empty-state-projectbuildstable']")
+        except:
+            self.fail(msg='Builds tab information is not present')
+
+        try:
+            self.driver.find_element_by_xpath("//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'importlayer"'+"]").click()
+            self.assertTrue(re.search("Import layer",self.driver.find_element_by_xpath("//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'importlayer"'+"]").text), 'Import layer tab in project topbar is misspelled')
+            self.driver.find_element_by_xpath("//fieldset[@id='repo-select']")
+            self.driver.find_element_by_xpath("//fieldset[@id='git-repo']")
+        except:
+            self.fail(msg='Import layer tab not loading properly')
+
+        try:
+            self.driver.find_element_by_xpath("//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'newcustomimage/"'+"]").click()
+            self.assertTrue(re.search("New custom image",self.driver.find_element_by_xpath("//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'newcustomimage/"'+"]").text), 'New custom image tab in project topbar is misspelled')
+            self.assertTrue(re.search("Select the image recipe you want to customise",self.driver.find_element_by_xpath("//div[@class='col-md-12']/h2").text),'The new custom image tab is not loading correctly')
+        except:
+            self.fail(msg='New custom image tab not loading properly')
+
+
+
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/tests/views/README b/import-layers/yocto-poky/bitbake/lib/toaster/tests/views/README
new file mode 100644
index 0000000..950c7c9
--- /dev/null
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/tests/views/README
@@ -0,0 +1,4 @@
+
+Django unit tests to verify classes and functions based on django Views
+
+To run just these tests use ./manage.py test tests.views
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/log/.create b/import-layers/yocto-poky/bitbake/lib/toaster/tests/views/__init__.py
similarity index 100%
copy from import-layers/yocto-poky/bitbake/lib/toaster/contrib/tts/log/.create
copy to import-layers/yocto-poky/bitbake/lib/toaster/tests/views/__init__.py
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/tests.py b/import-layers/yocto-poky/bitbake/lib/toaster/tests/views/test_views.py
similarity index 85%
rename from import-layers/yocto-poky/bitbake/lib/toaster/toastergui/tests.py
rename to import-layers/yocto-poky/bitbake/lib/toaster/tests/views/test_views.py
index 2b5894f..1463077 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/tests.py
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/tests/views/test_views.py
@@ -24,31 +24,26 @@
 from django.test import TestCase
 from django.test.client import RequestFactory
 from django.core.urlresolvers import reverse
-from django.utils import timezone
 from django.db.models import Q
 
-from orm.models import Project, Release, BitbakeVersion, Package, LogMessage
-from orm.models import LayerSource, Layer, Build
-from orm.models import Layer_Version, Recipe, Machine, ProjectLayer, Target
-from orm.models import CustomImageRecipe, ProjectVariable
+from orm.models import Project, Package
+from orm.models import Layer_Version, Recipe
+from orm.models import CustomImageRecipe
 from orm.models import CustomImagePackage
 
-import toastermain
 import inspect
 import toastergui
 
 from toastergui.tables import SoftwareRecipesTable
 import json
-from datetime import timedelta
 from bs4 import BeautifulSoup
-import re
 import string
-import json
 
 PROJECT_NAME = "test project"
 PROJECT_NAME2 = "test project 2"
 CLI_BUILDS_PROJECT_NAME = 'Command line builds'
 
+
 class ViewTests(TestCase):
     """Tests to verify view APIs."""
 
@@ -75,7 +70,8 @@
         url = reverse('all-projects')
         response = self.client.get(url, {"format": "json"}, follow=True)
         self.assertEqual(response.status_code, 200)
-        self.assertTrue(response['Content-Type'].startswith('application/json'))
+        self.assertTrue(response['Content-Type'].startswith(
+            'application/json'))
 
         data = json.loads(response.content.decode('utf-8'))
 
@@ -98,13 +94,13 @@
         urls = [layers_url,
                 prj_url,
                 reverse('xhr_recipestypeahead', args=(self.project.id,)),
-                reverse('xhr_machinestypeahead', args=(self.project.id,)),
-               ]
+                reverse('xhr_machinestypeahead', args=(self.project.id,))]
 
         def basic_reponse_check(response, url):
             """Check data structure of http response."""
             self.assertEqual(response.status_code, 200)
-            self.assertTrue(response['Content-Type'].startswith('application/json'))
+            self.assertTrue(response['Content-Type'].startswith(
+                'application/json'))
 
             data = json.loads(response.content.decode('utf-8'))
 
@@ -133,7 +129,6 @@
 
             return False
 
-
         for url in urls:
             results = False
 
@@ -147,35 +142,36 @@
             # from each of the urls
             self.assertTrue(results)
 
-    def test_xhr_import_layer(self):
-        """Test xhr_importlayer API"""
-        #Test for importing an already existing layer
-        args = {'vcs_url' : "git://git.example.com/test",
-                'name' : "base-layer",
-                'git_ref': "c12b9596afd236116b25ce26dbe0d793de9dc7ce",
-                'project_id': self.project.id,
-                'local_source_dir': "",
-                'dir_path' : "/path/in/repository"}
-        response = self.client.post(reverse('xhr_importlayer'), args)
+    def test_xhr_add_layer(self):
+        """Test xhr_add API"""
+        # Test for importing an already existing layer
+        api_url = reverse('xhr_layer', args=(self.project.id,))
+
+        layer_data = {'vcs_url': "git://git.example.com/test",
+                      'name': "base-layer",
+                      'git_ref': "c12b9596afd236116b25ce26dbe0d793de9dc7ce",
+                      'project_id': self.project.id,
+                      'local_source_dir': "",
+                      'add_to_project': True,
+                      'dir_path': "/path/in/repository"}
+
+        layer_data_json = json.dumps(layer_data)
+
+        response = self.client.put(api_url, layer_data_json)
         data = json.loads(response.content.decode('utf-8'))
         self.assertEqual(response.status_code, 200)
         self.assertEqual(data["error"], "ok")
 
-        #Test to verify import of a layer successful
-        args['name'] = "meta-oe"
-        response = self.client.post(reverse('xhr_importlayer'), args)
-        data = json.loads(response.content.decode('utf-8'))
-        self.assertTrue(data["error"], "ok")
+        self.assertTrue(
+            layer_data['name'] in
+            self.project.get_all_compatible_layer_versions().values_list(
+                'layer__name',
+                flat=True),
+            "Could not find imported layer in project's all layers list"
+        )
 
-        #Test for html tag in the data
-        args['<'] = "testing html tag"
-        response = self.client.post(reverse('xhr_importlayer'), args)
-        data = json.loads(response.content.decode('utf-8'))
-        self.assertNotEqual(data["error"], "ok")
-
-        #Empty data passed
-        args = {}
-        response = self.client.post(reverse('xhr_importlayer'), args)
+        # Empty data passed
+        response = self.client.put(api_url, "{}")
         data = json.loads(response.content.decode('utf-8'))
         self.assertNotEqual(data["error"], "ok")
 
@@ -232,15 +228,14 @@
                     "info": {'id': self.customr.id,
                              'name': self.customr.name,
                              'base_recipe_id': self.recipe1.id,
-                             'project_id': self.project.id,
-                            }
-                   }
-        self.assertEqual(json.loads(response.content.decode('utf-8')), expected)
+                             'project_id': self.project.id}}
+        self.assertEqual(json.loads(response.content.decode('utf-8')),
+                         expected)
 
     def test_xhr_custom_del(self):
         """Test deleting custom recipe"""
         name = "to be deleted"
-        recipe = CustomImageRecipe.objects.create(\
+        recipe = CustomImageRecipe.objects.create(
                      name=name, project=self.project,
                      base_recipe=self.recipe1,
                      file_path="/tmp/testing",
@@ -259,7 +254,8 @@
         url = reverse('xhr_customrecipe_id', args=(recipe.id,))
         response = self.client.delete(url)
         self.assertEqual(response.status_code, 200)
-        self.assertNotEqual(json.loads(response.content.decode('utf-8'))["error"], "ok")
+        self.assertNotEqual(json.loads(
+            response.content.decode('utf-8'))["error"], "ok")
 
     def test_xhr_custom_packages(self):
         """Test adding and deleting package to a custom recipe"""
@@ -280,7 +276,8 @@
 
         response = self.client.delete(del_url)
         self.assertEqual(response.status_code, 200)
-        self.assertEqual(json.loads(response.content.decode('utf-8')), {"error": "ok"})
+        self.assertEqual(json.loads(response.content.decode('utf-8')),
+                         {"error": "ok"})
         all_packages = self.customr.get_all_packages().values_list('pk',
                                                                    flat=True)
 
@@ -292,7 +289,8 @@
 
         response = self.client.delete(del_url)
         self.assertEqual(response.status_code, 200)
-        self.assertNotEqual(json.loads(response.content.decode('utf-8'))["error"], "ok")
+        self.assertNotEqual(json.loads(
+            response.content.decode('utf-8'))["error"], "ok")
 
     def test_xhr_custom_packages_err(self):
         """Test error conditions of xhr_customrecipe_packages"""
@@ -303,8 +301,9 @@
             for method in (self.client.put, self.client.delete):
                 response = method(url)
                 self.assertEqual(response.status_code, 200)
-                self.assertNotEqual(json.loads(response.content.decode('utf-8')),
-                                    {"error": "ok"})
+                self.assertNotEqual(json.loads(
+                    response.content.decode('utf-8')),
+                    {"error": "ok"})
 
     def test_download_custom_recipe(self):
         """Download the recipe file generated for the custom image"""
@@ -490,27 +489,28 @@
                         # filter string to pass as the option
                         # This is the name of the filter:action
                         # e.g. project_filter:not_in_project
-                        filter_string = "%s:%s" % (column['filter_name'],
-                                                   filter_action['action_name'])
+                        filter_string = "%s:%s" % (
+                            column['filter_name'],
+                            filter_action['action_name'])
                         # Now get the data with the filter applied
                         filtered_data = get_data(table_cls(),
-                                                 {"filter" : filter_string})
+                                                 {"filter": filter_string})
 
                         # date range filter actions can't specify the
                         # number of results they return, so their count is 0
-                        if filter_action['count'] != None:
-                            self.assertEqual(len(filtered_data['rows']),
-                                             int(filter_action['count']),
-                                             "We added a table filter for %s but "
-                                             "the number of rows returned was not "
-                                             "what the filter info said there "
-                                             "would be" % name)
-
+                        if filter_action['count'] is not None:
+                            self.assertEqual(
+                                len(filtered_data['rows']),
+                                int(filter_action['count']),
+                                "We added a table filter for %s but "
+                                "the number of rows returned was not "
+                                "what the filter info said there "
+                                "would be" % name)
 
             # Test search functionality on the table
             something_found = False
             for search in list(string.ascii_letters):
-                search_data = get_data(table_cls(), {'search' : search})
+                search_data = get_data(table_cls(), {'search': search})
 
                 if len(search_data['rows']) > 0:
                     something_found = True
@@ -521,20 +521,20 @@
                             " was found for the search of table %s" % name)
 
             # Test the limit functionality on the table
-            limited_data = get_data(table_cls(), {'limit' : "1"})
+            limited_data = get_data(table_cls(), {'limit': "1"})
             self.assertEqual(len(limited_data['rows']),
                              1,
                              "Limit 1 set on table %s but not 1 row returned"
                              % name)
 
             # Test the pagination functionality on the table
-            page_one_data = get_data(table_cls(), {'limit' : "1",
+            page_one_data = get_data(table_cls(), {'limit': "1",
                                                    "page": "1"})['rows'][0]
 
-            page_two_data = get_data(table_cls(), {'limit' : "1",
+            page_two_data = get_data(table_cls(), {'limit': "1",
                                                    "page": "2"})['rows'][0]
 
             self.assertNotEqual(page_one_data,
                                 page_two_data,
-                                "Changed page on table %s but first row is the "
-                                "same as the previous page" % name)
+                                "Changed page on table %s but first row is"
+                                " the same as the previous page" % name)
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/api.py b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/api.py
index ae1f150..1a6507c 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/api.py
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/api.py
@@ -20,6 +20,7 @@
 
 import re
 import logging
+import json
 from collections import Counter
 
 from orm.models import Project, ProjectTarget, Build, Layer_Version
@@ -136,14 +137,63 @@
 
 
 class XhrLayer(View):
-    """ Get and Update Layer information """
+    """ Delete, Get, Add and Update Layer information
+
+        Methods: GET POST DELETE PUT
+    """
+
+    def get(self, request, *args, **kwargs):
+        """
+        Get layer information
+
+        Method: GET
+        Entry point: /xhr_layer/<project id>/<layerversion_id>
+        """
+
+        try:
+            layer_version = Layer_Version.objects.get(
+                pk=kwargs['layerversion_id'])
+
+            project = Project.objects.get(pk=kwargs['pid'])
+
+            project_layers = ProjectLayer.objects.filter(
+                project=project).values_list("layercommit_id",
+                                             flat=True)
+
+            ret = {
+                'error': 'ok',
+                'id': layer_version.pk,
+                'name': layer_version.layer.name,
+                'layerdetailurl':
+                layer_version.get_detailspage_url(project.pk),
+                'vcs_ref': layer_version.get_vcs_reference(),
+                'vcs_url': layer_version.layer.vcs_url,
+                'local_source_dir': layer_version.layer.local_source_dir,
+                'layerdeps': {
+                    "list": [
+                        {
+                            "id": dep.id,
+                            "name": dep.layer.name,
+                            "layerdetailurl":
+                            dep.get_detailspage_url(project.pk),
+                            "vcs_url": dep.layer.vcs_url,
+                            "vcs_reference": dep.get_vcs_reference()
+                        }
+                        for dep in layer_version.get_alldeps(project.id)]
+                },
+                'projectlayers': list(project_layers)
+            }
+
+            return JsonResponse(ret)
+        except Layer_Version.DoesNotExist:
+            error_response("No such layer")
 
     def post(self, request, *args, **kwargs):
         """
           Update a layer
 
-          Entry point: /xhr_layer/<layerversion_id>
           Method: POST
+          Entry point: /xhr_layer/<layerversion_id>
 
           Args:
               vcs_url, dirpath, commit, up_branch, summary, description,
@@ -201,9 +251,100 @@
             return error_response("Could not update layer version entry: %s"
                                   % e)
 
-        return JsonResponse({"error": "ok"})
+        return error_response("ok")
+
+    def put(self, request, *args, **kwargs):
+        """ Add a new layer
+
+        Method: PUT
+        Entry point: /xhr_layer/<project id>/
+        Args:
+            project_id, name,
+            [vcs_url, dir_path, git_ref], [local_source_dir], [layer_deps
+            (csv)]
+
+        """
+        try:
+            project = Project.objects.get(pk=kwargs['pid'])
+
+            layer_data = json.loads(request.body.decode('utf-8'))
+
+            # We require a unique layer name as otherwise the lists of layers
+            # becomes very confusing
+            existing_layers = \
+                project.get_all_compatible_layer_versions().values_list(
+                    "layer__name",
+                    flat=True)
+
+            add_to_project = False
+            layer_deps_added = []
+            if 'add_to_project' in layer_data:
+                add_to_project = True
+
+            if layer_data['name'] in existing_layers:
+                return JsonResponse({"error": "layer-name-exists"})
+
+            layer = Layer.objects.create(name=layer_data['name'])
+
+            layer_version = Layer_Version.objects.create(
+                layer=layer,
+                project=project,
+                layer_source=LayerSource.TYPE_IMPORTED)
+
+            # Local layer
+            if ('local_source_dir' in layer_data) and layer.local_source_dir:
+                layer.local_source_dir = layer_data['local_source_dir']
+            # git layer
+            elif 'vcs_url' in layer_data:
+                layer.vcs_url = layer_data['vcs_url']
+                layer_version.dirpath = layer_data['dir_path']
+                layer_version.commit = layer_data['git_ref']
+                layer_version.branch = layer_data['git_ref']
+
+            layer.save()
+            layer_version.save()
+
+            if add_to_project:
+                ProjectLayer.objects.get_or_create(
+                    layercommit=layer_version, project=project)
+
+            # Add the layer dependencies
+            if 'layer_deps' in layer_data:
+                for layer_dep_id in layer_data['layer_deps'].split(","):
+                    layer_dep = Layer_Version.objects.get(pk=layer_dep_id)
+                    LayerVersionDependency.objects.get_or_create(
+                        layer_version=layer_version, depends_on=layer_dep)
+
+                    # Add layer deps to the project if specified
+                    if add_to_project:
+                        created, pl = ProjectLayer.objects.get_or_create(
+                            layercommit=layer_dep, project=project)
+                        layer_deps_added.append(
+                            {'name': layer_dep.layer.name,
+                             'layerdetailurl':
+                             layer_dep.get_detailspage_url(project.pk)})
+
+        except Layer_Version.DoesNotExist:
+            return error_response("layer-dep-not-found")
+        except Project.DoesNotExist:
+            return error_response("project-not-found")
+        except KeyError:
+            return error_response("incorrect-parameters")
+
+        return JsonResponse({'error': "ok",
+                             'imported_layer': {
+                                 'name': layer.name,
+                                 'layerdetailurl':
+                                 layer_version.get_detailspage_url()},
+                             'deps_added': layer_deps_added})
 
     def delete(self, request, *args, **kwargs):
+        """ Delete an imported layer
+
+        Method: DELETE
+        Entry point: /xhr_layer/<projed id>/<layerversion_id>
+
+        """
         try:
             # We currently only allow Imported layers to be deleted
             layer_version = Layer_Version.objects.get(
@@ -291,10 +432,13 @@
                 return error_response("recipe-already-exists")
 
             # create layer 'Custom layer' and verion if needed
-            layer = Layer.objects.get_or_create(
+            layer, l_created = Layer.objects.get_or_create(
                 name=CustomImageRecipe.LAYER_NAME,
-                summary="Layer for custom recipes",
-                vcs_url="file:///toaster_created_layer")[0]
+                summary="Layer for custom recipes")
+
+            if l_created:
+                layer.local_source_dir = "toaster_created_layer"
+                layer.save()
 
             # Check if we have a layer version already
             # We don't use get_or_create here because the dirpath will change
@@ -303,9 +447,10 @@
                                                 Q(layer=layer) &
                                                 Q(build=None)).last()
             if lver is None:
-                lver, created = Layer_Version.objects.get_or_create(
+                lver, lv_created = Layer_Version.objects.get_or_create(
                     project=params['project'],
                     layer=layer,
+                    layer_source=LayerSource.TYPE_LOCAL,
                     dirpath="toaster_created_layer")
 
             # Add a dependency on our layer to the base recipe's layer
@@ -319,7 +464,7 @@
                                                optional=False)
 
             # Create the actual recipe
-            recipe, created = CustomImageRecipe.objects.get_or_create(
+            recipe, r_created = CustomImageRecipe.objects.get_or_create(
                 name=request.POST["name"],
                 base_recipe=params["base"],
                 project=params["project"],
@@ -329,7 +474,7 @@
             # If we created the object then setup these fields. They may get
             # overwritten later on and cause the get_or_create to create a
             # duplicate if they've changed.
-            if created:
+            if r_created:
                 recipe.file_path = request.POST["name"]
                 recipe.license = "MIT"
                 recipe.version = "0.1"
@@ -789,6 +934,9 @@
                 "url": layer.layercommit.layer.layer_index_url,
                 "layerdetailurl": layer.layercommit.get_detailspage_url(
                     project.pk),
+                "xhrLayerUrl": reverse("xhr_layer",
+                                       args=(project.pk,
+                                             layer.layercommit.pk)),
                 "layersource": layer.layercommit.layer_source
             })
 
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/buildtables.py b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/buildtables.py
index dd0a690..755a7c2 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/buildtables.py
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/buildtables.py
@@ -571,6 +571,7 @@
         super(BuildTimeTable, self).setup_columns(**kwargs)
 
         self.columns[self.toggle_columns['order']]['hidden'] = True
+        self.columns[self.toggle_columns['order']]['hideable'] = True
         self.columns[self.toggle_columns['sstate_result']]['hidden'] = True
         self.columns[self.toggle_columns['elapsed_time']]['hidden'] = False
 
@@ -586,6 +587,7 @@
         super(BuildCPUTimeTable, self).setup_columns(**kwargs)
 
         self.columns[self.toggle_columns['order']]['hidden'] = True
+        self.columns[self.toggle_columns['order']]['hideable'] = True
         self.columns[self.toggle_columns['sstate_result']]['hidden'] = True
         self.columns[self.toggle_columns['cpu_time_sys']]['hidden'] = False
         self.columns[self.toggle_columns['cpu_time_user']]['hidden'] = False
@@ -602,5 +604,6 @@
         super(BuildIOTable, self).setup_columns(**kwargs)
 
         self.columns[self.toggle_columns['order']]['hidden'] = True
+        self.columns[self.toggle_columns['order']]['hideable'] = True
         self.columns[self.toggle_columns['sstate_result']]['hidden'] = True
         self.columns[self.toggle_columns['disk_io']]['hidden'] = False
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/css/default.css b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/css/default.css
index ff24e8c..5cd7e211 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/css/default.css
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/css/default.css
@@ -210,7 +210,6 @@
 #import-layer-name,
 #layer-subdir { width: 20%; }
 #layer-git-repo-url { width: 40%; }
-#layer-git-ref { width: 32%; }
 #local-dir-path { width: 45%; }
 #layer-dependency { width: 16em; }
 #layer-deps-list { margin-top: 0; }
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/css/prettify.css b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/css/prettify.css
deleted file mode 100755
index b317a7c..0000000
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/css/prettify.css
+++ /dev/null
@@ -1 +0,0 @@
-.pln{color:#000}@media screen{.str{color:#080}.kwd{color:#008}.com{color:#800}.typ{color:#606}.lit{color:#066}.pun,.opn,.clo{color:#660}.tag{color:#008}.atn{color:#606}.atv{color:#080}.dec,.var{color:#606}.fun{color:red}}@media print,projection{.str{color:#060}.kwd{color:#006;font-weight:bold}.com{color:#600;font-style:italic}.typ{color:#404;font-weight:bold}.lit{color:#044}.pun,.opn,.clo{color:#440}.tag{color:#006;font-weight:bold}.atn{color:#404}.atv{color:#060}}pre.prettyprint{padding:2px;border:1px solid #888}ol.linenums{margin-top:0;margin-bottom:0}li.L0,li.L1,li.L2,li.L3,li.L5,li.L6,li.L7,li.L8{list-style-type:none}li.L1,li.L3,li.L5,li.L7,li.L9{background:#eee}
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/customrecipe.js b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/customrecipe.js
index 9ea9602..8b1c190 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/customrecipe.js
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/customrecipe.js
@@ -312,5 +312,11 @@
     });
   });
 
+ /* Stop the download link from working if it is in disabled state
+  * http://getbootstrap.com/css/#forms-disabled-fieldsets
+  */
+ $("a[disabled=disabled]").click(function(e){
+   e.preventDefault();
+ });
 
 }
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/highlight.pack.js b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/highlight.pack.js
new file mode 100644
index 0000000..8cc886f
--- /dev/null
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/highlight.pack.js
@@ -0,0 +1,2 @@
+/*! highlight.js v9.12.0 | BSD3 License | git.io/hljslicense */
+!function(e){var n="object"==typeof window&&window||"object"==typeof self&&self;"undefined"!=typeof exports?e(exports):n&&(n.hljs=e({}),"function"==typeof define&&define.amd&&define([],function(){return n.hljs}))}(function(e){function n(e){return e.replace(/&/g,"&amp;").replace(/</g,"&lt;").replace(/>/g,"&gt;")}function t(e){return e.nodeName.toLowerCase()}function r(e,n){var t=e&&e.exec(n);return t&&0===t.index}function a(e){return k.test(e)}function i(e){var n,t,r,i,o=e.className+" ";if(o+=e.parentNode?e.parentNode.className:"",t=B.exec(o))return w(t[1])?t[1]:"no-highlight";for(o=o.split(/\s+/),n=0,r=o.length;r>n;n++)if(i=o[n],a(i)||w(i))return i}function o(e){var n,t={},r=Array.prototype.slice.call(arguments,1);for(n in e)t[n]=e[n];return r.forEach(function(e){for(n in e)t[n]=e[n]}),t}function u(e){var n=[];return function r(e,a){for(var i=e.firstChild;i;i=i.nextSibling)3===i.nodeType?a+=i.nodeValue.length:1===i.nodeType&&(n.push({event:"start",offset:a,node:i}),a=r(i,a),t(i).match(/br|hr|img|input/)||n.push({event:"stop",offset:a,node:i}));return a}(e,0),n}function c(e,r,a){function i(){return e.length&&r.length?e[0].offset!==r[0].offset?e[0].offset<r[0].offset?e:r:"start"===r[0].event?e:r:e.length?e:r}function o(e){function r(e){return" "+e.nodeName+'="'+n(e.value).replace('"',"&quot;")+'"'}s+="<"+t(e)+E.map.call(e.attributes,r).join("")+">"}function u(e){s+="</"+t(e)+">"}function c(e){("start"===e.event?o:u)(e.node)}for(var l=0,s="",f=[];e.length||r.length;){var g=i();if(s+=n(a.substring(l,g[0].offset)),l=g[0].offset,g===e){f.reverse().forEach(u);do c(g.splice(0,1)[0]),g=i();while(g===e&&g.length&&g[0].offset===l);f.reverse().forEach(o)}else"start"===g[0].event?f.push(g[0].node):f.pop(),c(g.splice(0,1)[0])}return s+n(a.substr(l))}function l(e){return e.v&&!e.cached_variants&&(e.cached_variants=e.v.map(function(n){return o(e,{v:null},n)})),e.cached_variants||e.eW&&[o(e)]||[e]}function s(e){function n(e){return e&&e.source||e}function t(t,r){return new RegExp(n(t),"m"+(e.cI?"i":"")+(r?"g":""))}function r(a,i){if(!a.compiled){if(a.compiled=!0,a.k=a.k||a.bK,a.k){var o={},u=function(n,t){e.cI&&(t=t.toLowerCase()),t.split(" ").forEach(function(e){var t=e.split("|");o[t[0]]=[n,t[1]?Number(t[1]):1]})};"string"==typeof a.k?u("keyword",a.k):x(a.k).forEach(function(e){u(e,a.k[e])}),a.k=o}a.lR=t(a.l||/\w+/,!0),i&&(a.bK&&(a.b="\\b("+a.bK.split(" ").join("|")+")\\b"),a.b||(a.b=/\B|\b/),a.bR=t(a.b),a.e||a.eW||(a.e=/\B|\b/),a.e&&(a.eR=t(a.e)),a.tE=n(a.e)||"",a.eW&&i.tE&&(a.tE+=(a.e?"|":"")+i.tE)),a.i&&(a.iR=t(a.i)),null==a.r&&(a.r=1),a.c||(a.c=[]),a.c=Array.prototype.concat.apply([],a.c.map(function(e){return l("self"===e?a:e)})),a.c.forEach(function(e){r(e,a)}),a.starts&&r(a.starts,i);var c=a.c.map(function(e){return e.bK?"\\.?("+e.b+")\\.?":e.b}).concat([a.tE,a.i]).map(n).filter(Boolean);a.t=c.length?t(c.join("|"),!0):{exec:function(){return null}}}}r(e)}function f(e,t,a,i){function o(e,n){var t,a;for(t=0,a=n.c.length;a>t;t++)if(r(n.c[t].bR,e))return n.c[t]}function u(e,n){if(r(e.eR,n)){for(;e.endsParent&&e.parent;)e=e.parent;return e}return e.eW?u(e.parent,n):void 0}function c(e,n){return!a&&r(n.iR,e)}function l(e,n){var t=N.cI?n[0].toLowerCase():n[0];return e.k.hasOwnProperty(t)&&e.k[t]}function p(e,n,t,r){var a=r?"":I.classPrefix,i='<span class="'+a,o=t?"":C;return i+=e+'">',i+n+o}function h(){var e,t,r,a;if(!E.k)return n(k);for(a="",t=0,E.lR.lastIndex=0,r=E.lR.exec(k);r;)a+=n(k.substring(t,r.index)),e=l(E,r),e?(B+=e[1],a+=p(e[0],n(r[0]))):a+=n(r[0]),t=E.lR.lastIndex,r=E.lR.exec(k);return a+n(k.substr(t))}function d(){var e="string"==typeof E.sL;if(e&&!y[E.sL])return n(k);var t=e?f(E.sL,k,!0,x[E.sL]):g(k,E.sL.length?E.sL:void 0);return E.r>0&&(B+=t.r),e&&(x[E.sL]=t.top),p(t.language,t.value,!1,!0)}function b(){L+=null!=E.sL?d():h(),k=""}function v(e){L+=e.cN?p(e.cN,"",!0):"",E=Object.create(e,{parent:{value:E}})}function m(e,n){if(k+=e,null==n)return b(),0;var t=o(n,E);if(t)return t.skip?k+=n:(t.eB&&(k+=n),b(),t.rB||t.eB||(k=n)),v(t,n),t.rB?0:n.length;var r=u(E,n);if(r){var a=E;a.skip?k+=n:(a.rE||a.eE||(k+=n),b(),a.eE&&(k=n));do E.cN&&(L+=C),E.skip||(B+=E.r),E=E.parent;while(E!==r.parent);return r.starts&&v(r.starts,""),a.rE?0:n.length}if(c(n,E))throw new Error('Illegal lexeme "'+n+'" for mode "'+(E.cN||"<unnamed>")+'"');return k+=n,n.length||1}var N=w(e);if(!N)throw new Error('Unknown language: "'+e+'"');s(N);var R,E=i||N,x={},L="";for(R=E;R!==N;R=R.parent)R.cN&&(L=p(R.cN,"",!0)+L);var k="",B=0;try{for(var M,j,O=0;;){if(E.t.lastIndex=O,M=E.t.exec(t),!M)break;j=m(t.substring(O,M.index),M[0]),O=M.index+j}for(m(t.substr(O)),R=E;R.parent;R=R.parent)R.cN&&(L+=C);return{r:B,value:L,language:e,top:E}}catch(T){if(T.message&&-1!==T.message.indexOf("Illegal"))return{r:0,value:n(t)};throw T}}function g(e,t){t=t||I.languages||x(y);var r={r:0,value:n(e)},a=r;return t.filter(w).forEach(function(n){var t=f(n,e,!1);t.language=n,t.r>a.r&&(a=t),t.r>r.r&&(a=r,r=t)}),a.language&&(r.second_best=a),r}function p(e){return I.tabReplace||I.useBR?e.replace(M,function(e,n){return I.useBR&&"\n"===e?"<br>":I.tabReplace?n.replace(/\t/g,I.tabReplace):""}):e}function h(e,n,t){var r=n?L[n]:t,a=[e.trim()];return e.match(/\bhljs\b/)||a.push("hljs"),-1===e.indexOf(r)&&a.push(r),a.join(" ").trim()}function d(e){var n,t,r,o,l,s=i(e);a(s)||(I.useBR?(n=document.createElementNS("http://www.w3.org/1999/xhtml","div"),n.innerHTML=e.innerHTML.replace(/\n/g,"").replace(/<br[ \/]*>/g,"\n")):n=e,l=n.textContent,r=s?f(s,l,!0):g(l),t=u(n),t.length&&(o=document.createElementNS("http://www.w3.org/1999/xhtml","div"),o.innerHTML=r.value,r.value=c(t,u(o),l)),r.value=p(r.value),e.innerHTML=r.value,e.className=h(e.className,s,r.language),e.result={language:r.language,re:r.r},r.second_best&&(e.second_best={language:r.second_best.language,re:r.second_best.r}))}function b(e){I=o(I,e)}function v(){if(!v.called){v.called=!0;var e=document.querySelectorAll("pre code");E.forEach.call(e,d)}}function m(){addEventListener("DOMContentLoaded",v,!1),addEventListener("load",v,!1)}function N(n,t){var r=y[n]=t(e);r.aliases&&r.aliases.forEach(function(e){L[e]=n})}function R(){return x(y)}function w(e){return e=(e||"").toLowerCase(),y[e]||y[L[e]]}var E=[],x=Object.keys,y={},L={},k=/^(no-?highlight|plain|text)$/i,B=/\blang(?:uage)?-([\w-]+)\b/i,M=/((^(<[^>]+>|\t|)+|(?:\n)))/gm,C="</span>",I={classPrefix:"hljs-",tabReplace:null,useBR:!1,languages:void 0};return e.highlight=f,e.highlightAuto=g,e.fixMarkup=p,e.highlightBlock=d,e.configure=b,e.initHighlighting=v,e.initHighlightingOnLoad=m,e.registerLanguage=N,e.listLanguages=R,e.getLanguage=w,e.inherit=o,e.IR="[a-zA-Z]\\w*",e.UIR="[a-zA-Z_]\\w*",e.NR="\\b\\d+(\\.\\d+)?",e.CNR="(-?)(\\b0[xX][a-fA-F0-9]+|(\\b\\d+(\\.\\d*)?|\\.\\d+)([eE][-+]?\\d+)?)",e.BNR="\\b(0b[01]+)",e.RSR="!|!=|!==|%|%=|&|&&|&=|\\*|\\*=|\\+|\\+=|,|-|-=|/=|/|:|;|<<|<<=|<=|<|===|==|=|>>>=|>>=|>=|>>>|>>|>|\\?|\\[|\\{|\\(|\\^|\\^=|\\||\\|=|\\|\\||~",e.BE={b:"\\\\[\\s\\S]",r:0},e.ASM={cN:"string",b:"'",e:"'",i:"\\n",c:[e.BE]},e.QSM={cN:"string",b:'"',e:'"',i:"\\n",c:[e.BE]},e.PWM={b:/\b(a|an|the|are|I'm|isn't|don't|doesn't|won't|but|just|should|pretty|simply|enough|gonna|going|wtf|so|such|will|you|your|they|like|more)\b/},e.C=function(n,t,r){var a=e.inherit({cN:"comment",b:n,e:t,c:[]},r||{});return a.c.push(e.PWM),a.c.push({cN:"doctag",b:"(?:TODO|FIXME|NOTE|BUG|XXX):",r:0}),a},e.CLCM=e.C("//","$"),e.CBCM=e.C("/\\*","\\*/"),e.HCM=e.C("#","$"),e.NM={cN:"number",b:e.NR,r:0},e.CNM={cN:"number",b:e.CNR,r:0},e.BNM={cN:"number",b:e.BNR,r:0},e.CSSNM={cN:"number",b:e.NR+"(%|em|ex|ch|rem|vw|vh|vmin|vmax|cm|mm|in|pt|pc|px|deg|grad|rad|turn|s|ms|Hz|kHz|dpi|dpcm|dppx)?",r:0},e.RM={cN:"regexp",b:/\//,e:/\/[gimuy]*/,i:/\n/,c:[e.BE,{b:/\[/,e:/\]/,r:0,c:[e.BE]}]},e.TM={cN:"title",b:e.IR,r:0},e.UTM={cN:"title",b:e.UIR,r:0},e.METHOD_GUARD={b:"\\.\\s*"+e.UIR,r:0},e});hljs.registerLanguage("sql",function(e){var t=e.C("--","$");return{cI:!0,i:/[<>{}*#]/,c:[{bK:"begin end start commit rollback savepoint lock alter create drop rename call delete do handler insert load replace select truncate update set show pragma grant merge describe use explain help declare prepare execute deallocate release unlock purge reset change stop analyze cache flush optimize repair kill install uninstall checksum restore check backup revoke comment",e:/;/,eW:!0,l:/[\w\.]+/,k:{keyword:"abort abs absolute acc acce accep accept access accessed accessible account acos action activate add addtime admin administer advanced advise aes_decrypt aes_encrypt after agent aggregate ali alia alias allocate allow alter always analyze ancillary and any anydata anydataset anyschema anytype apply archive archived archivelog are as asc ascii asin assembly assertion associate asynchronous at atan atn2 attr attri attrib attribu attribut attribute attributes audit authenticated authentication authid authors auto autoallocate autodblink autoextend automatic availability avg backup badfile basicfile before begin beginning benchmark between bfile bfile_base big bigfile bin binary_double binary_float binlog bit_and bit_count bit_length bit_or bit_xor bitmap blob_base block blocksize body both bound buffer_cache buffer_pool build bulk by byte byteordermark bytes cache caching call calling cancel capacity cascade cascaded case cast catalog category ceil ceiling chain change changed char_base char_length character_length characters characterset charindex charset charsetform charsetid check checksum checksum_agg child choose chr chunk class cleanup clear client clob clob_base clone close cluster_id cluster_probability cluster_set clustering coalesce coercibility col collate collation collect colu colum column column_value columns columns_updated comment commit compact compatibility compiled complete composite_limit compound compress compute concat concat_ws concurrent confirm conn connec connect connect_by_iscycle connect_by_isleaf connect_by_root connect_time connection consider consistent constant constraint constraints constructor container content contents context contributors controlfile conv convert convert_tz corr corr_k corr_s corresponding corruption cos cost count count_big counted covar_pop covar_samp cpu_per_call cpu_per_session crc32 create creation critical cross cube cume_dist curdate current current_date current_time current_timestamp current_user cursor curtime customdatum cycle data database databases datafile datafiles datalength date_add date_cache date_format date_sub dateadd datediff datefromparts datename datepart datetime2fromparts day day_to_second dayname dayofmonth dayofweek dayofyear days db_role_change dbtimezone ddl deallocate declare decode decompose decrement decrypt deduplicate def defa defau defaul default defaults deferred defi defin define degrees delayed delegate delete delete_all delimited demand dense_rank depth dequeue des_decrypt des_encrypt des_key_file desc descr descri describ describe descriptor deterministic diagnostics difference dimension direct_load directory disable disable_all disallow disassociate discardfile disconnect diskgroup distinct distinctrow distribute distributed div do document domain dotnet double downgrade drop dumpfile duplicate duration each edition editionable editions element ellipsis else elsif elt empty enable enable_all enclosed encode encoding encrypt end end-exec endian enforced engine engines enqueue enterprise entityescaping eomonth error errors escaped evalname evaluate event eventdata events except exception exceptions exchange exclude excluding execu execut execute exempt exists exit exp expire explain export export_set extended extent external external_1 external_2 externally extract failed failed_login_attempts failover failure far fast feature_set feature_value fetch field fields file file_name_convert filesystem_like_logging final finish first first_value fixed flash_cache flashback floor flush following follows for forall force form forma format found found_rows freelist freelists freepools fresh from from_base64 from_days ftp full function general generated get get_format get_lock getdate getutcdate global global_name globally go goto grant grants greatest group group_concat group_id grouping grouping_id groups gtid_subtract guarantee guard handler hash hashkeys having hea head headi headin heading heap help hex hierarchy high high_priority hosts hour http id ident_current ident_incr ident_seed identified identity idle_time if ifnull ignore iif ilike ilm immediate import in include including increment index indexes indexing indextype indicator indices inet6_aton inet6_ntoa inet_aton inet_ntoa infile initial initialized initially initrans inmemory inner innodb input insert install instance instantiable instr interface interleaved intersect into invalidate invisible is is_free_lock is_ipv4 is_ipv4_compat is_not is_not_null is_used_lock isdate isnull isolation iterate java join json json_exists keep keep_duplicates key keys kill language large last last_day last_insert_id last_value lax lcase lead leading least leaves left len lenght length less level levels library like like2 like4 likec limit lines link list listagg little ln load load_file lob lobs local localtime localtimestamp locate locator lock locked log log10 log2 logfile logfiles logging logical logical_reads_per_call logoff logon logs long loop low low_priority lower lpad lrtrim ltrim main make_set makedate maketime managed management manual map mapping mask master master_pos_wait match matched materialized max maxextents maximize maxinstances maxlen maxlogfiles maxloghistory maxlogmembers maxsize maxtrans md5 measures median medium member memcompress memory merge microsecond mid migration min minextents minimum mining minus minute minvalue missing mod mode model modification modify module monitoring month months mount move movement multiset mutex name name_const names nan national native natural nav nchar nclob nested never new newline next nextval no no_write_to_binlog noarchivelog noaudit nobadfile nocheck nocompress nocopy nocycle nodelay nodiscardfile noentityescaping noguarantee nokeep nologfile nomapping nomaxvalue nominimize nominvalue nomonitoring none noneditionable nonschema noorder nopr nopro noprom nopromp noprompt norely noresetlogs noreverse normal norowdependencies noschemacheck noswitch not nothing notice notrim novalidate now nowait nth_value nullif nulls num numb numbe nvarchar nvarchar2 object ocicoll ocidate ocidatetime ociduration ociinterval ociloblocator ocinumber ociref ocirefcursor ocirowid ocistring ocitype oct octet_length of off offline offset oid oidindex old on online only opaque open operations operator optimal optimize option optionally or oracle oracle_date oradata ord ordaudio orddicom orddoc order ordimage ordinality ordvideo organization orlany orlvary out outer outfile outline output over overflow overriding package pad parallel parallel_enable parameters parent parse partial partition partitions pascal passing password password_grace_time password_lock_time password_reuse_max password_reuse_time password_verify_function patch path patindex pctincrease pctthreshold pctused pctversion percent percent_rank percentile_cont percentile_disc performance period period_add period_diff permanent physical pi pipe pipelined pivot pluggable plugin policy position post_transaction pow power pragma prebuilt precedes preceding precision prediction prediction_cost prediction_details prediction_probability prediction_set prepare present preserve prior priority private private_sga privileges procedural procedure procedure_analyze processlist profiles project prompt protection public publishingservername purge quarter query quick quiesce quota quotename radians raise rand range rank raw read reads readsize rebuild record records recover recovery recursive recycle redo reduced ref reference referenced references referencing refresh regexp_like register regr_avgx regr_avgy regr_count regr_intercept regr_r2 regr_slope regr_sxx regr_sxy reject rekey relational relative relaylog release release_lock relies_on relocate rely rem remainder rename repair repeat replace replicate replication required reset resetlogs resize resource respect restore restricted result result_cache resumable resume retention return returning returns reuse reverse revoke right rlike role roles rollback rolling rollup round row row_count rowdependencies rowid rownum rows rtrim rules safe salt sample save savepoint sb1 sb2 sb4 scan schema schemacheck scn scope scroll sdo_georaster sdo_topo_geometry search sec_to_time second section securefile security seed segment select self sequence sequential serializable server servererror session session_user sessions_per_user set sets settings sha sha1 sha2 share shared shared_pool short show shrink shutdown si_averagecolor si_colorhistogram si_featurelist si_positionalcolor si_stillimage si_texture siblings sid sign sin size size_t sizes skip slave sleep smalldatetimefromparts smallfile snapshot some soname sort soundex source space sparse spfile split sql sql_big_result sql_buffer_result sql_cache sql_calc_found_rows sql_small_result sql_variant_property sqlcode sqldata sqlerror sqlname sqlstate sqrt square standalone standby start starting startup statement static statistics stats_binomial_test stats_crosstab stats_ks_test stats_mode stats_mw_test stats_one_way_anova stats_t_test_ stats_t_test_indep stats_t_test_one stats_t_test_paired stats_wsr_test status std stddev stddev_pop stddev_samp stdev stop storage store stored str str_to_date straight_join strcmp strict string struct stuff style subdate subpartition subpartitions substitutable substr substring subtime subtring_index subtype success sum suspend switch switchoffset switchover sync synchronous synonym sys sys_xmlagg sysasm sysaux sysdate sysdatetimeoffset sysdba sysoper system system_user sysutcdatetime table tables tablespace tan tdo template temporary terminated tertiary_weights test than then thread through tier ties time time_format time_zone timediff timefromparts timeout timestamp timestampadd timestampdiff timezone_abbr timezone_minute timezone_region to to_base64 to_date to_days to_seconds todatetimeoffset trace tracking transaction transactional translate translation treat trigger trigger_nestlevel triggers trim truncate try_cast try_convert try_parse type ub1 ub2 ub4 ucase unarchived unbounded uncompress under undo unhex unicode uniform uninstall union unique unix_timestamp unknown unlimited unlock unpivot unrecoverable unsafe unsigned until untrusted unusable unused update updated upgrade upped upper upsert url urowid usable usage use use_stored_outlines user user_data user_resources users using utc_date utc_timestamp uuid uuid_short validate validate_password_strength validation valist value values var var_samp varcharc vari varia variab variabl variable variables variance varp varraw varrawc varray verify version versions view virtual visible void wait wallet warning warnings week weekday weekofyear wellformed when whene whenev wheneve whenever where while whitespace with within without work wrapped xdb xml xmlagg xmlattributes xmlcast xmlcolattval xmlelement xmlexists xmlforest xmlindex xmlnamespaces xmlpi xmlquery xmlroot xmlschema xmlserialize xmltable xmltype xor year year_to_month years yearweek",literal:"true false null",built_in:"array bigint binary bit blob boolean char character date dec decimal float int int8 integer interval number numeric real record serial serial8 smallint text varchar varying void"},c:[{cN:"string",b:"'",e:"'",c:[e.BE,{b:"''"}]},{cN:"string",b:'"',e:'"',c:[e.BE,{b:'""'}]},{cN:"string",b:"`",e:"`",c:[e.BE]},e.CNM,e.CBCM,t]},e.CBCM,t]}});hljs.registerLanguage("perl",function(e){var t="getpwent getservent quotemeta msgrcv scalar kill dbmclose undef lc ma syswrite tr send umask sysopen shmwrite vec qx utime local oct semctl localtime readpipe do return format read sprintf dbmopen pop getpgrp not getpwnam rewinddir qqfileno qw endprotoent wait sethostent bless s|0 opendir continue each sleep endgrent shutdown dump chomp connect getsockname die socketpair close flock exists index shmgetsub for endpwent redo lstat msgctl setpgrp abs exit select print ref gethostbyaddr unshift fcntl syscall goto getnetbyaddr join gmtime symlink semget splice x|0 getpeername recv log setsockopt cos last reverse gethostbyname getgrnam study formline endhostent times chop length gethostent getnetent pack getprotoent getservbyname rand mkdir pos chmod y|0 substr endnetent printf next open msgsnd readdir use unlink getsockopt getpriority rindex wantarray hex system getservbyport endservent int chr untie rmdir prototype tell listen fork shmread ucfirst setprotoent else sysseek link getgrgid shmctl waitpid unpack getnetbyname reset chdir grep split require caller lcfirst until warn while values shift telldir getpwuid my getprotobynumber delete and sort uc defined srand accept package seekdir getprotobyname semop our rename seek if q|0 chroot sysread setpwent no crypt getc chown sqrt write setnetent setpriority foreach tie sin msgget map stat getlogin unless elsif truncate exec keys glob tied closedirioctl socket readlink eval xor readline binmode setservent eof ord bind alarm pipe atan2 getgrent exp time push setgrent gt lt or ne m|0 break given say state when",r={cN:"subst",b:"[$@]\\{",e:"\\}",k:t},s={b:"->{",e:"}"},n={v:[{b:/\$\d/},{b:/[\$%@](\^\w\b|#\w+(::\w+)*|{\w+}|\w+(::\w*)*)/},{b:/[\$%@][^\s\w{]/,r:0}]},i=[e.BE,r,n],o=[n,e.HCM,e.C("^\\=\\w","\\=cut",{eW:!0}),s,{cN:"string",c:i,v:[{b:"q[qwxr]?\\s*\\(",e:"\\)",r:5},{b:"q[qwxr]?\\s*\\[",e:"\\]",r:5},{b:"q[qwxr]?\\s*\\{",e:"\\}",r:5},{b:"q[qwxr]?\\s*\\|",e:"\\|",r:5},{b:"q[qwxr]?\\s*\\<",e:"\\>",r:5},{b:"qw\\s+q",e:"q",r:5},{b:"'",e:"'",c:[e.BE]},{b:'"',e:'"'},{b:"`",e:"`",c:[e.BE]},{b:"{\\w+}",c:[],r:0},{b:"-?\\w+\\s*\\=\\>",c:[],r:0}]},{cN:"number",b:"(\\b0[0-7_]+)|(\\b0x[0-9a-fA-F_]+)|(\\b[1-9][0-9_]*(\\.[0-9_]+)?)|[0_]\\b",r:0},{b:"(\\/\\/|"+e.RSR+"|\\b(split|return|print|reverse|grep)\\b)\\s*",k:"split return print reverse grep",r:0,c:[e.HCM,{cN:"regexp",b:"(s|tr|y)/(\\\\.|[^/])*/(\\\\.|[^/])*/[a-z]*",r:10},{cN:"regexp",b:"(m|qr)?/",e:"/[a-z]*",c:[e.BE],r:0}]},{cN:"function",bK:"sub",e:"(\\s*\\(.*?\\))?[;{]",eE:!0,r:5,c:[e.TM]},{b:"-\\w\\b",r:0},{b:"^__DATA__$",e:"^__END__$",sL:"mojolicious",c:[{b:"^@@.*",e:"$",cN:"comment"}]}];return r.c=o,s.c=o,{aliases:["pl","pm"],l:/[\w\.]+/,k:t,c:o}});hljs.registerLanguage("ini",function(e){var b={cN:"string",c:[e.BE],v:[{b:"'''",e:"'''",r:10},{b:'"""',e:'"""',r:10},{b:'"',e:'"'},{b:"'",e:"'"}]};return{aliases:["toml"],cI:!0,i:/\S/,c:[e.C(";","$"),e.HCM,{cN:"section",b:/^\s*\[+/,e:/\]+/},{b:/^[a-z0-9\[\]_-]+\s*=\s*/,e:"$",rB:!0,c:[{cN:"attr",b:/[a-z0-9\[\]_-]+/},{b:/=/,eW:!0,r:0,c:[{cN:"literal",b:/\bon|off|true|false|yes|no\b/},{cN:"variable",v:[{b:/\$[\w\d"][\w\d_]*/},{b:/\$\{(.*?)}/}]},b,{cN:"number",b:/([\+\-]+)?[\d]+_[\d_]+/},e.NM]}]}]}});hljs.registerLanguage("diff",function(e){return{aliases:["patch"],c:[{cN:"meta",r:10,v:[{b:/^@@ +\-\d+,\d+ +\+\d+,\d+ +@@$/},{b:/^\*\*\* +\d+,\d+ +\*\*\*\*$/},{b:/^\-\-\- +\d+,\d+ +\-\-\-\-$/}]},{cN:"comment",v:[{b:/Index: /,e:/$/},{b:/={3,}/,e:/$/},{b:/^\-{3}/,e:/$/},{b:/^\*{3} /,e:/$/},{b:/^\+{3}/,e:/$/},{b:/\*{5}/,e:/\*{5}$/}]},{cN:"addition",b:"^\\+",e:"$"},{cN:"deletion",b:"^\\-",e:"$"},{cN:"addition",b:"^\\!",e:"$"}]}});hljs.registerLanguage("bash",function(e){var t={cN:"variable",v:[{b:/\$[\w\d#@][\w\d_]*/},{b:/\$\{(.*?)}/}]},s={cN:"string",b:/"/,e:/"/,c:[e.BE,t,{cN:"variable",b:/\$\(/,e:/\)/,c:[e.BE]}]},a={cN:"string",b:/'/,e:/'/};return{aliases:["sh","zsh"],l:/\b-?[a-z\._]+\b/,k:{keyword:"if then else elif fi for while in do done case esac function",literal:"true false",built_in:"break cd continue eval exec exit export getopts hash pwd readonly return shift test times trap umask unset alias bind builtin caller command declare echo enable help let local logout mapfile printf read readarray source type typeset ulimit unalias set shopt autoload bg bindkey bye cap chdir clone comparguments compcall compctl compdescribe compfiles compgroups compquote comptags comptry compvalues dirs disable disown echotc echoti emulate fc fg float functions getcap getln history integer jobs kill limit log noglob popd print pushd pushln rehash sched setcap setopt stat suspend ttyctl unfunction unhash unlimit unsetopt vared wait whence where which zcompile zformat zftp zle zmodload zparseopts zprof zpty zregexparse zsocket zstyle ztcp",_:"-ne -eq -lt -gt -f -d -e -s -l -a"},c:[{cN:"meta",b:/^#![^\n]+sh\s*$/,r:10},{cN:"function",b:/\w[\w\d_]*\s*\(\s*\)\s*\{/,rB:!0,c:[e.inherit(e.TM,{b:/\w[\w\d_]*/})],r:0},e.HCM,s,a,t]}});hljs.registerLanguage("php",function(e){var c={b:"\\$+[a-zA-Z_-ÿ][a-zA-Z0-9_-ÿ]*"},i={cN:"meta",b:/<\?(php)?|\?>/},t={cN:"string",c:[e.BE,i],v:[{b:'b"',e:'"'},{b:"b'",e:"'"},e.inherit(e.ASM,{i:null}),e.inherit(e.QSM,{i:null})]},a={v:[e.BNM,e.CNM]};return{aliases:["php3","php4","php5","php6"],cI:!0,k:"and include_once list abstract global private echo interface as static endswitch array null if endwhile or const for endforeach self var while isset public protected exit foreach throw elseif include __FILE__ empty require_once do xor return parent clone use __CLASS__ __LINE__ else break print eval new catch __METHOD__ case exception default die require __FUNCTION__ enddeclare final try switch continue endfor endif declare unset true false trait goto instanceof insteadof __DIR__ __NAMESPACE__ yield finally",c:[e.HCM,e.C("//","$",{c:[i]}),e.C("/\\*","\\*/",{c:[{cN:"doctag",b:"@[A-Za-z]+"}]}),e.C("__halt_compiler.+?;",!1,{eW:!0,k:"__halt_compiler",l:e.UIR}),{cN:"string",b:/<<<['"]?\w+['"]?$/,e:/^\w+;?$/,c:[e.BE,{cN:"subst",v:[{b:/\$\w+/},{b:/\{\$/,e:/\}/}]}]},i,{cN:"keyword",b:/\$this\b/},c,{b:/(::|->)+[a-zA-Z_\x7f-\xff][a-zA-Z0-9_\x7f-\xff]*/},{cN:"function",bK:"function",e:/[;{]/,eE:!0,i:"\\$|\\[|%",c:[e.UTM,{cN:"params",b:"\\(",e:"\\)",c:["self",c,e.CBCM,t,a]}]},{cN:"class",bK:"class interface",e:"{",eE:!0,i:/[:\(\$"]/,c:[{bK:"extends implements"},e.UTM]},{bK:"namespace",e:";",i:/[\.']/,c:[e.UTM]},{bK:"use",e:";",c:[e.UTM]},{b:"=>"},t,a]}});hljs.registerLanguage("python",function(e){var r={keyword:"and elif is global as in if from raise for except finally print import pass return exec else break not with class assert yield try while continue del or def lambda async await nonlocal|10 None True False",built_in:"Ellipsis NotImplemented"},b={cN:"meta",b:/^(>>>|\.\.\.) /},c={cN:"subst",b:/\{/,e:/\}/,k:r,i:/#/},a={cN:"string",c:[e.BE],v:[{b:/(u|b)?r?'''/,e:/'''/,c:[b],r:10},{b:/(u|b)?r?"""/,e:/"""/,c:[b],r:10},{b:/(fr|rf|f)'''/,e:/'''/,c:[b,c]},{b:/(fr|rf|f)"""/,e:/"""/,c:[b,c]},{b:/(u|r|ur)'/,e:/'/,r:10},{b:/(u|r|ur)"/,e:/"/,r:10},{b:/(b|br)'/,e:/'/},{b:/(b|br)"/,e:/"/},{b:/(fr|rf|f)'/,e:/'/,c:[c]},{b:/(fr|rf|f)"/,e:/"/,c:[c]},e.ASM,e.QSM]},s={cN:"number",r:0,v:[{b:e.BNR+"[lLjJ]?"},{b:"\\b(0o[0-7]+)[lLjJ]?"},{b:e.CNR+"[lLjJ]?"}]},i={cN:"params",b:/\(/,e:/\)/,c:["self",b,s,a]};return c.c=[a,s,b],{aliases:["py","gyp"],k:r,i:/(<\/|->|\?)|=>/,c:[b,s,a,e.HCM,{v:[{cN:"function",bK:"def"},{cN:"class",bK:"class"}],e:/:/,i:/[${=;\n,]/,c:[e.UTM,i,{b:/->/,eW:!0,k:"None"}]},{cN:"meta",b:/^[\t ]*@/,e:/$/},{b:/\b(print|exec)\(/}]}});hljs.registerLanguage("coffeescript",function(e){var c={keyword:"in if for while finally new do return else break catch instanceof throw try this switch continue typeof delete debugger super yield import export from as default await then unless until loop of by when and or is isnt not",literal:"true false null undefined yes no on off",built_in:"npm require console print module global window document"},n="[A-Za-z$_][0-9A-Za-z$_]*",r={cN:"subst",b:/#\{/,e:/}/,k:c},i=[e.BNM,e.inherit(e.CNM,{starts:{e:"(\\s*/)?",r:0}}),{cN:"string",v:[{b:/'''/,e:/'''/,c:[e.BE]},{b:/'/,e:/'/,c:[e.BE]},{b:/"""/,e:/"""/,c:[e.BE,r]},{b:/"/,e:/"/,c:[e.BE,r]}]},{cN:"regexp",v:[{b:"///",e:"///",c:[r,e.HCM]},{b:"//[gim]*",r:0},{b:/\/(?![ *])(\\\/|.)*?\/[gim]*(?=\W|$)/}]},{b:"@"+n},{sL:"javascript",eB:!0,eE:!0,v:[{b:"```",e:"```"},{b:"`",e:"`"}]}];r.c=i;var s=e.inherit(e.TM,{b:n}),t="(\\(.*\\))?\\s*\\B[-=]>",o={cN:"params",b:"\\([^\\(]",rB:!0,c:[{b:/\(/,e:/\)/,k:c,c:["self"].concat(i)}]};return{aliases:["coffee","cson","iced"],k:c,i:/\/\*/,c:i.concat([e.C("###","###"),e.HCM,{cN:"function",b:"^\\s*"+n+"\\s*=\\s*"+t,e:"[-=]>",rB:!0,c:[s,o]},{b:/[:\(,=]\s*/,r:0,c:[{cN:"function",b:t,e:"[-=]>",rB:!0,c:[o]}]},{cN:"class",bK:"class",e:"$",i:/[:="\[\]]/,c:[{bK:"extends",eW:!0,i:/[:="\[\]]/,c:[s]},s]},{b:n+":",e:":",rB:!0,rE:!0,r:0}])}});hljs.registerLanguage("cpp",function(t){var e={cN:"keyword",b:"\\b[a-z\\d_]*_t\\b"},r={cN:"string",v:[{b:'(u8?|U)?L?"',e:'"',i:"\\n",c:[t.BE]},{b:'(u8?|U)?R"',e:'"',c:[t.BE]},{b:"'\\\\?.",e:"'",i:"."}]},s={cN:"number",v:[{b:"\\b(0b[01']+)"},{b:"(-?)\\b([\\d']+(\\.[\\d']*)?|\\.[\\d']+)(u|U|l|L|ul|UL|f|F|b|B)"},{b:"(-?)(\\b0[xX][a-fA-F0-9']+|(\\b[\\d']+(\\.[\\d']*)?|\\.[\\d']+)([eE][-+]?[\\d']+)?)"}],r:0},i={cN:"meta",b:/#\s*[a-z]+\b/,e:/$/,k:{"meta-keyword":"if else elif endif define undef warning error line pragma ifdef ifndef include"},c:[{b:/\\\n/,r:0},t.inherit(r,{cN:"meta-string"}),{cN:"meta-string",b:/<[^\n>]*>/,e:/$/,i:"\\n"},t.CLCM,t.CBCM]},a=t.IR+"\\s*\\(",c={keyword:"int float while private char catch import module export virtual operator sizeof dynamic_cast|10 typedef const_cast|10 const for static_cast|10 union namespace unsigned long volatile static protected bool template mutable if public friend do goto auto void enum else break extern using asm case typeid short reinterpret_cast|10 default double register explicit signed typename try this switch continue inline delete alignof constexpr decltype noexcept static_assert thread_local restrict _Bool complex _Complex _Imaginary atomic_bool atomic_char atomic_schar atomic_uchar atomic_short atomic_ushort atomic_int atomic_uint atomic_long atomic_ulong atomic_llong atomic_ullong new throw return and or not",built_in:"std string cin cout cerr clog stdin stdout stderr stringstream istringstream ostringstream auto_ptr deque list queue stack vector map set bitset multiset multimap unordered_set unordered_map unordered_multiset unordered_multimap array shared_ptr abort abs acos asin atan2 atan calloc ceil cosh cos exit exp fabs floor fmod fprintf fputs free frexp fscanf isalnum isalpha iscntrl isdigit isgraph islower isprint ispunct isspace isupper isxdigit tolower toupper labs ldexp log10 log malloc realloc memchr memcmp memcpy memset modf pow printf putchar puts scanf sinh sin snprintf sprintf sqrt sscanf strcat strchr strcmp strcpy strcspn strlen strncat strncmp strncpy strpbrk strrchr strspn strstr tanh tan vfprintf vprintf vsprintf endl initializer_list unique_ptr",literal:"true false nullptr NULL"},n=[e,t.CLCM,t.CBCM,s,r];return{aliases:["c","cc","h","c++","h++","hpp"],k:c,i:"</",c:n.concat([i,{b:"\\b(deque|list|queue|stack|vector|map|set|bitset|multiset|multimap|unordered_map|unordered_set|unordered_multiset|unordered_multimap|array)\\s*<",e:">",k:c,c:["self",e]},{b:t.IR+"::",k:c},{v:[{b:/=/,e:/;/},{b:/\(/,e:/\)/},{bK:"new throw return else",e:/;/}],k:c,c:n.concat([{b:/\(/,e:/\)/,k:c,c:n.concat(["self"]),r:0}]),r:0},{cN:"function",b:"("+t.IR+"[\\*&\\s]+)+"+a,rB:!0,e:/[{;=]/,eE:!0,k:c,i:/[^\w\s\*&]/,c:[{b:a,rB:!0,c:[t.TM],r:0},{cN:"params",b:/\(/,e:/\)/,k:c,r:0,c:[t.CLCM,t.CBCM,r,s,e]},t.CLCM,t.CBCM,i]},{cN:"class",bK:"class struct",e:/[{;:]/,c:[{b:/</,e:/>/,c:["self"]},t.TM]}]),exports:{preprocessor:i,strings:r,k:c}}});hljs.registerLanguage("cs",function(e){var i={keyword:"abstract as base bool break byte case catch char checked const continue decimal default delegate do double enum event explicit extern finally fixed float for foreach goto if implicit in int interface internal is lock long nameof object operator out override params private protected public readonly ref sbyte sealed short sizeof stackalloc static string struct switch this try typeof uint ulong unchecked unsafe ushort using virtual void volatile while add alias ascending async await by descending dynamic equals from get global group into join let on orderby partial remove select set value var where yield",literal:"null false true"},t={cN:"string",b:'@"',e:'"',c:[{b:'""'}]},r=e.inherit(t,{i:/\n/}),a={cN:"subst",b:"{",e:"}",k:i},c=e.inherit(a,{i:/\n/}),n={cN:"string",b:/\$"/,e:'"',i:/\n/,c:[{b:"{{"},{b:"}}"},e.BE,c]},s={cN:"string",b:/\$@"/,e:'"',c:[{b:"{{"},{b:"}}"},{b:'""'},a]},o=e.inherit(s,{i:/\n/,c:[{b:"{{"},{b:"}}"},{b:'""'},c]});a.c=[s,n,t,e.ASM,e.QSM,e.CNM,e.CBCM],c.c=[o,n,r,e.ASM,e.QSM,e.CNM,e.inherit(e.CBCM,{i:/\n/})];var l={v:[s,n,t,e.ASM,e.QSM]},b=e.IR+"(<"+e.IR+"(\\s*,\\s*"+e.IR+")*>)?(\\[\\])?";return{aliases:["csharp"],k:i,i:/::/,c:[e.C("///","$",{rB:!0,c:[{cN:"doctag",v:[{b:"///",r:0},{b:"<!--|-->"},{b:"</?",e:">"}]}]}),e.CLCM,e.CBCM,{cN:"meta",b:"#",e:"$",k:{"meta-keyword":"if else elif endif define undef warning error line region endregion pragma checksum"}},l,e.CNM,{bK:"class interface",e:/[{;=]/,i:/[^\s:]/,c:[e.TM,e.CLCM,e.CBCM]},{bK:"namespace",e:/[{;=]/,i:/[^\s:]/,c:[e.inherit(e.TM,{b:"[a-zA-Z](\\.?\\w)*"}),e.CLCM,e.CBCM]},{cN:"meta",b:"^\\s*\\[",eB:!0,e:"\\]",eE:!0,c:[{cN:"meta-string",b:/"/,e:/"/}]},{bK:"new return throw await else",r:0},{cN:"function",b:"("+b+"\\s+)+"+e.IR+"\\s*\\(",rB:!0,e:/[{;=]/,eE:!0,k:i,c:[{b:e.IR+"\\s*\\(",rB:!0,c:[e.TM],r:0},{cN:"params",b:/\(/,e:/\)/,eB:!0,eE:!0,k:i,r:0,c:[l,e.CNM,e.CBCM]},e.CLCM,e.CBCM]}]}});hljs.registerLanguage("shell",function(s){return{aliases:["console"],c:[{cN:"meta",b:"^\\s{0,3}[\\w\\d\\[\\]()@-]*[>%$#]",starts:{e:"$",sL:"bash"}}]}});hljs.registerLanguage("ruby",function(e){var b="[a-zA-Z_]\\w*[!?=]?|[-+~]\\@|<<|>>|=~|===?|<=>|[<>]=?|\\*\\*|[-/+%^&*~`|]|\\[\\]=?",r={keyword:"and then defined module in return redo if BEGIN retry end for self when next until do begin unless END rescue else break undef not super class case require yield alias while ensure elsif or include attr_reader attr_writer attr_accessor",literal:"true false nil"},c={cN:"doctag",b:"@[A-Za-z]+"},a={b:"#<",e:">"},s=[e.C("#","$",{c:[c]}),e.C("^\\=begin","^\\=end",{c:[c],r:10}),e.C("^__END__","\\n$")],n={cN:"subst",b:"#\\{",e:"}",k:r},t={cN:"string",c:[e.BE,n],v:[{b:/'/,e:/'/},{b:/"/,e:/"/},{b:/`/,e:/`/},{b:"%[qQwWx]?\\(",e:"\\)"},{b:"%[qQwWx]?\\[",e:"\\]"},{b:"%[qQwWx]?{",e:"}"},{b:"%[qQwWx]?<",e:">"},{b:"%[qQwWx]?/",e:"/"},{b:"%[qQwWx]?%",e:"%"},{b:"%[qQwWx]?-",e:"-"},{b:"%[qQwWx]?\\|",e:"\\|"},{b:/\B\?(\\\d{1,3}|\\x[A-Fa-f0-9]{1,2}|\\u[A-Fa-f0-9]{4}|\\?\S)\b/},{b:/<<(-?)\w+$/,e:/^\s*\w+$/}]},i={cN:"params",b:"\\(",e:"\\)",endsParent:!0,k:r},d=[t,a,{cN:"class",bK:"class module",e:"$|;",i:/=/,c:[e.inherit(e.TM,{b:"[A-Za-z_]\\w*(::\\w+)*(\\?|\\!)?"}),{b:"<\\s*",c:[{b:"("+e.IR+"::)?"+e.IR}]}].concat(s)},{cN:"function",bK:"def",e:"$|;",c:[e.inherit(e.TM,{b:b}),i].concat(s)},{b:e.IR+"::"},{cN:"symbol",b:e.UIR+"(\\!|\\?)?:",r:0},{cN:"symbol",b:":(?!\\s)",c:[t,{b:b}],r:0},{cN:"number",b:"(\\b0[0-7_]+)|(\\b0x[0-9a-fA-F_]+)|(\\b[1-9][0-9_]*(\\.[0-9_]+)?)|[0_]\\b",r:0},{b:"(\\$\\W)|((\\$|\\@\\@?)(\\w+))"},{cN:"params",b:/\|/,e:/\|/,k:r},{b:"("+e.RSR+"|unless)\\s*",k:"unless",c:[a,{cN:"regexp",c:[e.BE,n],i:/\n/,v:[{b:"/",e:"/[a-z]*"},{b:"%r{",e:"}[a-z]*"},{b:"%r\\(",e:"\\)[a-z]*"},{b:"%r!",e:"![a-z]*"},{b:"%r\\[",e:"\\][a-z]*"}]}].concat(s),r:0}].concat(s);n.c=d,i.c=d;var l="[>?]>",o="[\\w#]+\\(\\w+\\):\\d+:\\d+>",u="(\\w+-)?\\d+\\.\\d+\\.\\d(p\\d+)?[^>]+>",w=[{b:/^\s*=>/,starts:{e:"$",c:d}},{cN:"meta",b:"^("+l+"|"+o+"|"+u+")",starts:{e:"$",c:d}}];return{aliases:["rb","gemspec","podspec","thor","irb"],k:r,i:/\/\*/,c:s.concat(w).concat(d)}});hljs.registerLanguage("nginx",function(e){var r={cN:"variable",v:[{b:/\$\d+/},{b:/\$\{/,e:/}/},{b:"[\\$\\@]"+e.UIR}]},b={eW:!0,l:"[a-z/_]+",k:{literal:"on off yes no true false none blocked debug info notice warn error crit select break last permanent redirect kqueue rtsig epoll poll /dev/poll"},r:0,i:"=>",c:[e.HCM,{cN:"string",c:[e.BE,r],v:[{b:/"/,e:/"/},{b:/'/,e:/'/}]},{b:"([a-z]+):/",e:"\\s",eW:!0,eE:!0,c:[r]},{cN:"regexp",c:[e.BE,r],v:[{b:"\\s\\^",e:"\\s|{|;",rE:!0},{b:"~\\*?\\s+",e:"\\s|{|;",rE:!0},{b:"\\*(\\.[a-z\\-]+)+"},{b:"([a-z\\-]+\\.)+\\*"}]},{cN:"number",b:"\\b\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}(:\\d{1,5})?\\b"},{cN:"number",b:"\\b\\d+[kKmMgGdshdwy]*\\b",r:0},r]};return{aliases:["nginxconf"],c:[e.HCM,{b:e.UIR+"\\s+{",rB:!0,e:"{",c:[{cN:"section",b:e.UIR}],r:0},{b:e.UIR+"\\s",e:";|{",rB:!0,c:[{cN:"attribute",b:e.UIR,starts:b}],r:0}],i:"[^\\s\\}]"}});hljs.registerLanguage("css",function(e){var c="[a-zA-Z-][a-zA-Z0-9_-]*",t={b:/[A-Z\_\.\-]+\s*:/,rB:!0,e:";",eW:!0,c:[{cN:"attribute",b:/\S/,e:":",eE:!0,starts:{eW:!0,eE:!0,c:[{b:/[\w-]+\(/,rB:!0,c:[{cN:"built_in",b:/[\w-]+/},{b:/\(/,e:/\)/,c:[e.ASM,e.QSM]}]},e.CSSNM,e.QSM,e.ASM,e.CBCM,{cN:"number",b:"#[0-9A-Fa-f]+"},{cN:"meta",b:"!important"}]}}]};return{cI:!0,i:/[=\/|'\$]/,c:[e.CBCM,{cN:"selector-id",b:/#[A-Za-z0-9_-]+/},{cN:"selector-class",b:/\.[A-Za-z0-9_-]+/},{cN:"selector-attr",b:/\[/,e:/\]/,i:"$"},{cN:"selector-pseudo",b:/:(:)?[a-zA-Z0-9\_\-\+\(\)"'.]+/},{b:"@(font-face|page)",l:"[a-z-]+",k:"font-face page"},{b:"@",e:"[{;]",i:/:/,c:[{cN:"keyword",b:/\w+/},{b:/\s/,eW:!0,eE:!0,r:0,c:[e.ASM,e.QSM,e.CSSNM]}]},{cN:"selector-tag",b:c,r:0},{b:"{",e:"}",i:/\S/,c:[e.CBCM,t]}]}});hljs.registerLanguage("makefile",function(e){var i={cN:"variable",v:[{b:"\\$\\("+e.UIR+"\\)",c:[e.BE]},{b:/\$[@%<?\^\+\*]/}]},r={cN:"string",b:/"/,e:/"/,c:[e.BE,i]},a={cN:"variable",b:/\$\([\w-]+\s/,e:/\)/,k:{built_in:"subst patsubst strip findstring filter filter-out sort word wordlist firstword lastword dir notdir suffix basename addsuffix addprefix join wildcard realpath abspath error warning shell origin flavor foreach if or and call eval file value"},c:[i]},n={b:"^"+e.UIR+"\\s*[:+?]?=",i:"\\n",rB:!0,c:[{b:"^"+e.UIR,e:"[:+?]?=",eE:!0}]},t={cN:"meta",b:/^\.PHONY:/,e:/$/,k:{"meta-keyword":".PHONY"},l:/[\.\w]+/},l={cN:"section",b:/^[^\s]+:/,e:/$/,c:[i]};return{aliases:["mk","mak"],k:"define endef undefine ifdef ifndef ifeq ifneq else endif include -include sinclude override export unexport private vpath",l:/[\w-]+/,c:[e.HCM,i,r,a,n,t,l]}});hljs.registerLanguage("java",function(e){var a="[À-ʸa-zA-Z_$][À-ʸa-zA-Z_$0-9]*",t=a+"(<"+a+"(\\s*,\\s*"+a+")*>)?",r="false synchronized int abstract float private char boolean static null if const for true while long strictfp finally protected import native final void enum else break transient catch instanceof byte super volatile case assert short package default double public try this switch continue throws protected public private module requires exports do",s="\\b(0[bB]([01]+[01_]+[01]+|[01]+)|0[xX]([a-fA-F0-9]+[a-fA-F0-9_]+[a-fA-F0-9]+|[a-fA-F0-9]+)|(([\\d]+[\\d_]+[\\d]+|[\\d]+)(\\.([\\d]+[\\d_]+[\\d]+|[\\d]+))?|\\.([\\d]+[\\d_]+[\\d]+|[\\d]+))([eE][-+]?\\d+)?)[lLfF]?",c={cN:"number",b:s,r:0};return{aliases:["jsp"],k:r,i:/<\/|#/,c:[e.C("/\\*\\*","\\*/",{r:0,c:[{b:/\w+@/,r:0},{cN:"doctag",b:"@[A-Za-z]+"}]}),e.CLCM,e.CBCM,e.ASM,e.QSM,{cN:"class",bK:"class interface",e:/[{;=]/,eE:!0,k:"class interface",i:/[:"\[\]]/,c:[{bK:"extends implements"},e.UTM]},{bK:"new throw return else",r:0},{cN:"function",b:"("+t+"\\s+)+"+e.UIR+"\\s*\\(",rB:!0,e:/[{;=]/,eE:!0,k:r,c:[{b:e.UIR+"\\s*\\(",rB:!0,r:0,c:[e.UTM]},{cN:"params",b:/\(/,e:/\)/,k:r,r:0,c:[e.ASM,e.QSM,e.CNM,e.CBCM]},e.CLCM,e.CBCM]},c,{cN:"meta",b:"@[A-Za-z]+"}]}});hljs.registerLanguage("http",function(e){var t="HTTP/[0-9\\.]+";return{aliases:["https"],i:"\\S",c:[{b:"^"+t,e:"$",c:[{cN:"number",b:"\\b\\d{3}\\b"}]},{b:"^[A-Z]+ (.*?) "+t+"$",rB:!0,e:"$",c:[{cN:"string",b:" ",e:" ",eB:!0,eE:!0},{b:t},{cN:"keyword",b:"[A-Z]+"}]},{cN:"attribute",b:"^\\w",e:": ",eE:!0,i:"\\n|\\s|=",starts:{e:"$",r:0}},{b:"\\n\\n",starts:{sL:[],eW:!0}}]}});hljs.registerLanguage("objectivec",function(e){var t={cN:"built_in",b:"\\b(AV|CA|CF|CG|CI|CL|CM|CN|CT|MK|MP|MTK|MTL|NS|SCN|SK|UI|WK|XC)\\w+"},_={keyword:"int float while char export sizeof typedef const struct for union unsigned long volatile static bool mutable if do return goto void enum else break extern asm case short default double register explicit signed typename this switch continue wchar_t inline readonly assign readwrite self @synchronized id typeof nonatomic super unichar IBOutlet IBAction strong weak copy in out inout bycopy byref oneway __strong __weak __block __autoreleasing @private @protected @public @try @property @end @throw @catch @finally @autoreleasepool @synthesize @dynamic @selector @optional @required @encode @package @import @defs @compatibility_alias __bridge __bridge_transfer __bridge_retained __bridge_retain __covariant __contravariant __kindof _Nonnull _Nullable _Null_unspecified __FUNCTION__ __PRETTY_FUNCTION__ __attribute__ getter setter retain unsafe_unretained nonnull nullable null_unspecified null_resettable class instancetype NS_DESIGNATED_INITIALIZER NS_UNAVAILABLE NS_REQUIRES_SUPER NS_RETURNS_INNER_POINTER NS_INLINE NS_AVAILABLE NS_DEPRECATED NS_ENUM NS_OPTIONS NS_SWIFT_UNAVAILABLE NS_ASSUME_NONNULL_BEGIN NS_ASSUME_NONNULL_END NS_REFINED_FOR_SWIFT NS_SWIFT_NAME NS_SWIFT_NOTHROW NS_DURING NS_HANDLER NS_ENDHANDLER NS_VALUERETURN NS_VOIDRETURN",literal:"false true FALSE TRUE nil YES NO NULL",built_in:"BOOL dispatch_once_t dispatch_queue_t dispatch_sync dispatch_async dispatch_once"},i=/[a-zA-Z@][a-zA-Z0-9_]*/,n="@interface @class @protocol @implementation";return{aliases:["mm","objc","obj-c"],k:_,l:i,i:"</",c:[t,e.CLCM,e.CBCM,e.CNM,e.QSM,{cN:"string",v:[{b:'@"',e:'"',i:"\\n",c:[e.BE]},{b:"'",e:"[^\\\\]'",i:"[^\\\\][^']"}]},{cN:"meta",b:"#",e:"$",c:[{cN:"meta-string",v:[{b:'"',e:'"'},{b:"<",e:">"}]}]},{cN:"class",b:"("+n.split(" ").join("|")+")\\b",e:"({|$)",eE:!0,k:n,l:i,c:[e.UTM]},{b:"\\."+e.UIR,r:0}]}});hljs.registerLanguage("javascript",function(e){var r="[A-Za-z$_][0-9A-Za-z$_]*",t={keyword:"in of if for while finally var new function do return void else break catch instanceof with throw case default try this switch continue typeof delete let yield const export super debugger as async await static import from as",literal:"true false null undefined NaN Infinity",built_in:"eval isFinite isNaN parseFloat parseInt decodeURI decodeURIComponent encodeURI encodeURIComponent escape unescape Object Function Boolean Error EvalError InternalError RangeError ReferenceError StopIteration SyntaxError TypeError URIError Number Math Date String RegExp Array Float32Array Float64Array Int16Array Int32Array Int8Array Uint16Array Uint32Array Uint8Array Uint8ClampedArray ArrayBuffer DataView JSON Intl arguments require module console window document Symbol Set Map WeakSet WeakMap Proxy Reflect Promise"},a={cN:"number",v:[{b:"\\b(0[bB][01]+)"},{b:"\\b(0[oO][0-7]+)"},{b:e.CNR}],r:0},n={cN:"subst",b:"\\$\\{",e:"\\}",k:t,c:[]},c={cN:"string",b:"`",e:"`",c:[e.BE,n]};n.c=[e.ASM,e.QSM,c,a,e.RM];var s=n.c.concat([e.CBCM,e.CLCM]);return{aliases:["js","jsx"],k:t,c:[{cN:"meta",r:10,b:/^\s*['"]use (strict|asm)['"]/},{cN:"meta",b:/^#!/,e:/$/},e.ASM,e.QSM,c,e.CLCM,e.CBCM,a,{b:/[{,]\s*/,r:0,c:[{b:r+"\\s*:",rB:!0,r:0,c:[{cN:"attr",b:r,r:0}]}]},{b:"("+e.RSR+"|\\b(case|return|throw)\\b)\\s*",k:"return throw case",c:[e.CLCM,e.CBCM,e.RM,{cN:"function",b:"(\\(.*?\\)|"+r+")\\s*=>",rB:!0,e:"\\s*=>",c:[{cN:"params",v:[{b:r},{b:/\(\s*\)/},{b:/\(/,e:/\)/,eB:!0,eE:!0,k:t,c:s}]}]},{b:/</,e:/(\/\w+|\w+\/)>/,sL:"xml",c:[{b:/<\w+\s*\/>/,skip:!0},{b:/<\w+/,e:/(\/\w+|\w+\/)>/,skip:!0,c:[{b:/<\w+\s*\/>/,skip:!0},"self"]}]}],r:0},{cN:"function",bK:"function",e:/\{/,eE:!0,c:[e.inherit(e.TM,{b:r}),{cN:"params",b:/\(/,e:/\)/,eB:!0,eE:!0,c:s}],i:/\[|%/},{b:/\$[(.]/},e.METHOD_GUARD,{cN:"class",bK:"class",e:/[{;=]/,eE:!0,i:/[:"\[\]]/,c:[{bK:"extends"},e.UTM]},{bK:"constructor",e:/\{/,eE:!0}],i:/#(?!!)/}});hljs.registerLanguage("apache",function(e){var r={cN:"number",b:"[\\$%]\\d+"};return{aliases:["apacheconf"],cI:!0,c:[e.HCM,{cN:"section",b:"</?",e:">"},{cN:"attribute",b:/\w+/,r:0,k:{nomarkup:"order deny allow setenv rewriterule rewriteengine rewritecond documentroot sethandler errordocument loadmodule options header listen serverroot servername"},starts:{e:/$/,r:0,k:{literal:"on off all"},c:[{cN:"meta",b:"\\s\\[",e:"\\]$"},{cN:"variable",b:"[\\$%]\\{",e:"\\}",c:["self",r]},r,e.QSM]}}],i:/\S/}});hljs.registerLanguage("xml",function(s){var e="[A-Za-z0-9\\._:-]+",t={eW:!0,i:/</,r:0,c:[{cN:"attr",b:e,r:0},{b:/=\s*/,r:0,c:[{cN:"string",endsParent:!0,v:[{b:/"/,e:/"/},{b:/'/,e:/'/},{b:/[^\s"'=<>`]+/}]}]}]};return{aliases:["html","xhtml","rss","atom","xjb","xsd","xsl","plist"],cI:!0,c:[{cN:"meta",b:"<!DOCTYPE",e:">",r:10,c:[{b:"\\[",e:"\\]"}]},s.C("<!--","-->",{r:10}),{b:"<\\!\\[CDATA\\[",e:"\\]\\]>",r:10},{b:/<\?(php)?/,e:/\?>/,sL:"php",c:[{b:"/\\*",e:"\\*/",skip:!0}]},{cN:"tag",b:"<style(?=\\s|>|$)",e:">",k:{name:"style"},c:[t],starts:{e:"</style>",rE:!0,sL:["css","xml"]}},{cN:"tag",b:"<script(?=\\s|>|$)",e:">",k:{name:"script"},c:[t],starts:{e:"</script>",rE:!0,sL:["actionscript","javascript","handlebars","xml"]}},{cN:"meta",v:[{b:/<\?xml/,e:/\?>/,r:10},{b:/<\?\w+/,e:/\?>/}]},{cN:"tag",b:"</?",e:"/?>",c:[{cN:"name",b:/[^\/><\s]+/,r:0},t]}]}});hljs.registerLanguage("markdown",function(e){return{aliases:["md","mkdown","mkd"],c:[{cN:"section",v:[{b:"^#{1,6}",e:"$"},{b:"^.+?\\n[=-]{2,}$"}]},{b:"<",e:">",sL:"xml",r:0},{cN:"bullet",b:"^([*+-]|(\\d+\\.))\\s+"},{cN:"strong",b:"[*_]{2}.+?[*_]{2}"},{cN:"emphasis",v:[{b:"\\*.+?\\*"},{b:"_.+?_",r:0}]},{cN:"quote",b:"^>\\s+",e:"$"},{cN:"code",v:[{b:"^```w*s*$",e:"^```s*$"},{b:"`.+?`"},{b:"^( {4}|	)",e:"$",r:0}]},{b:"^[-\\*]{3,}",e:"$"},{b:"\\[.+?\\][\\(\\[].*?[\\)\\]]",rB:!0,c:[{cN:"string",b:"\\[",e:"\\]",eB:!0,rE:!0,r:0},{cN:"link",b:"\\]\\(",e:"\\)",eB:!0,eE:!0},{cN:"symbol",b:"\\]\\[",e:"\\]",eB:!0,eE:!0}],r:10},{b:/^\[[^\n]+\]:/,rB:!0,c:[{cN:"symbol",b:/\[/,e:/\]/,eB:!0,eE:!0},{cN:"link",b:/:\s*/,e:/$/,eB:!0}]}]}});hljs.registerLanguage("json",function(e){var i={literal:"true false null"},n=[e.QSM,e.CNM],r={e:",",eW:!0,eE:!0,c:n,k:i},t={b:"{",e:"}",c:[{cN:"attr",b:/"/,e:/"/,c:[e.BE],i:"\\n"},e.inherit(r,{b:/:/})],i:"\\S"},c={b:"\\[",e:"\\]",c:[e.inherit(r)],i:"\\S"};return n.splice(n.length,0,t,c),{c:n,k:i,i:"\\S"}});
\ No newline at end of file
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/importlayer.js b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/importlayer.js
index 30dc282..2964839 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/importlayer.js
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/importlayer.js
@@ -45,7 +45,7 @@
     function(layer) {
     if (layer.results.length > 0) {
       currentLayerDepSelection = layer.results[0];
-      layerDepBtn.click();
+        layerDepBtn.click();
     }
   });
 
@@ -158,6 +158,7 @@
         project_id: libtoaster.ctx.projectId,
         layer_deps: layerDepsCsv,
         local_source_dir: $('#local-dir-path').val(),
+        add_to_project: true,
       };
 
       if ($('input[name=repo]:checked').val() == "git") {
@@ -168,13 +169,15 @@
       }
 
       $.ajax({
-          type: "POST",
-          url: ctx.xhrImportLayerUrl,
-          data: layerData,
+          type: "PUT",
+          url: ctx.xhrLayerUrl,
+          data: JSON.stringify(layerData),
           headers: { 'X-CSRFToken' : $.cookie('csrftoken')},
           success: function (data) {
             if (data.error != "ok") {
               console.log(data.error);
+              /* let the user know why nothing happened */
+              alert(data.error)
             } else {
               createImportedNotification(data);
               window.location.replace(libtoaster.ctx.projectPageUrl);
@@ -243,9 +246,18 @@
         enable_import_btn(true);
       }
 
-      if ($("#git-repo-radio").prop("checked") &&
-          vcsURLInput.val().length > 0 && gitRefInput.val().length > 0) {
-        enable_import_btn(true);
+      if ($("#git-repo-radio").prop("checked")) {
+        if (gitRefInput.val().length > 0 &&
+            gitRefInput.val() == 'HEAD') {
+          $('#invalid-layer-revision-hint').show();
+          $('#layer-revision-ctrl').addClass('has-error');
+          enable_import_btn(false);
+        } else if (vcsURLInput.val().length > 0 &&
+                   gitRefInput.val().length > 0) {
+          $('#invalid-layer-revision-hint').hide();
+          $('#layer-revision-ctrl').removeClass('has-error');
+          enable_import_btn(true);
+        }
       }
     }
 
@@ -332,19 +344,36 @@
     check_form();
   });
 
-  /* Have a guess at the layer name */
+  /* Setup 'blank' typeahead */
+  libtoaster.makeTypeahead(gitRefInput,
+                           ctx.xhrGitRevTypeAheadUrl,
+                           { git_url: null }, function(){});
+
+
   vcsURLInput.focusout(function (){
+    if (!$(this).val())
+      return;
+
     /* If we a layer name specified don't overwrite it or if there isn't a
      * url typed in yet return
      */
-    if (layerNameInput.val() || !$(this).val())
-      return;
-
-    if ($(this).val().search("/")){
+    if (!layerNameInput.val() && $(this).val().search("/")){
       var urlPts = $(this).val().split("/");
+      /* Add a suggestion of the layer name */
       var suggestion = urlPts[urlPts.length-1].replace(".git","");
       layerNameInput.val(suggestion);
     }
+
+    /* Now actually setup the typeahead properly with the git url entered */
+    gitRefInput._typeahead('destroy');
+
+    libtoaster.makeTypeahead(gitRefInput,
+                             ctx.xhrGitRevTypeAheadUrl,
+                             { git_url: $(this).val() },
+                             function(selected){
+                               gitRefInput._typeahead("close");
+                             });
+
   });
 
   function radioDisplay() {
@@ -389,7 +418,7 @@
       var input = $(this);
       var reBeginWithSlash = /^\//;
       var reCheckVariable = /^\$/;
-      var re = /([ <>\\|":\.%\?\*]+)/;
+      var re = /([ <>\\|":%\?\*]+)/;
 
       var invalidDir = re.test(input.val());
       var invalidSlash = reBeginWithSlash.test(input.val());
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/libtoaster.js b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/libtoaster.js
index 86662b7..6f9b5d0 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/libtoaster.js
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/libtoaster.js
@@ -274,9 +274,13 @@
   }
 
   function _addRmLayer(layerObj, add, doneCb){
+    if (layerObj.xhrLayerUrl === undefined){
+      throw("xhrLayerUrl is undefined")
+    }
+
     if (add === true) {
       /* If adding get the deps for this layer */
-      libtoaster.getLayerDepsForProject(layerObj.layerdetailurl,
+      libtoaster.getLayerDepsForProject(layerObj.xhrLayerUrl,
         function (layers) {
 
         /* got result for dependencies */
@@ -542,11 +546,9 @@
     }
 
     /*
-     * PrettyPrint plugin.
-     *
+     * highlight plugin.
      */
-    // Init
-    prettyPrint();
+    hljs.initHighlightingOnLoad();
 
     // Prevent invalid links from jumping page scroll
     $('a[href=#]').click(function() {
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/prettify.js b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/prettify.js
deleted file mode 100755
index eef5ad7..0000000
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/prettify.js
+++ /dev/null
@@ -1,28 +0,0 @@
-var q=null;window.PR_SHOULD_USE_CONTINUATION=!0;
-(function(){function L(a){function m(a){var f=a.charCodeAt(0);if(f!==92)return f;var b=a.charAt(1);return(f=r[b])?f:"0"<=b&&b<="7"?parseInt(a.substring(1),8):b==="u"||b==="x"?parseInt(a.substring(2),16):a.charCodeAt(1)}function e(a){if(a<32)return(a<16?"\\x0":"\\x")+a.toString(16);a=String.fromCharCode(a);if(a==="\\"||a==="-"||a==="["||a==="]")a="\\"+a;return a}function h(a){for(var f=a.substring(1,a.length-1).match(/\\u[\dA-Fa-f]{4}|\\x[\dA-Fa-f]{2}|\\[0-3][0-7]{0,2}|\\[0-7]{1,2}|\\[\S\s]|[^\\]/g),a=
-[],b=[],o=f[0]==="^",c=o?1:0,i=f.length;c<i;++c){var j=f[c];if(/\\[bdsw]/i.test(j))a.push(j);else{var j=m(j),d;c+2<i&&"-"===f[c+1]?(d=m(f[c+2]),c+=2):d=j;b.push([j,d]);d<65||j>122||(d<65||j>90||b.push([Math.max(65,j)|32,Math.min(d,90)|32]),d<97||j>122||b.push([Math.max(97,j)&-33,Math.min(d,122)&-33]))}}b.sort(function(a,f){return a[0]-f[0]||f[1]-a[1]});f=[];j=[NaN,NaN];for(c=0;c<b.length;++c)i=b[c],i[0]<=j[1]+1?j[1]=Math.max(j[1],i[1]):f.push(j=i);b=["["];o&&b.push("^");b.push.apply(b,a);for(c=0;c<
-f.length;++c)i=f[c],b.push(e(i[0])),i[1]>i[0]&&(i[1]+1>i[0]&&b.push("-"),b.push(e(i[1])));b.push("]");return b.join("")}function y(a){for(var f=a.source.match(/\[(?:[^\\\]]|\\[\S\s])*]|\\u[\dA-Fa-f]{4}|\\x[\dA-Fa-f]{2}|\\\d+|\\[^\dux]|\(\?[!:=]|[()^]|[^()[\\^]+/g),b=f.length,d=[],c=0,i=0;c<b;++c){var j=f[c];j==="("?++i:"\\"===j.charAt(0)&&(j=+j.substring(1))&&j<=i&&(d[j]=-1)}for(c=1;c<d.length;++c)-1===d[c]&&(d[c]=++t);for(i=c=0;c<b;++c)j=f[c],j==="("?(++i,d[i]===void 0&&(f[c]="(?:")):"\\"===j.charAt(0)&&
-(j=+j.substring(1))&&j<=i&&(f[c]="\\"+d[i]);for(i=c=0;c<b;++c)"^"===f[c]&&"^"!==f[c+1]&&(f[c]="");if(a.ignoreCase&&s)for(c=0;c<b;++c)j=f[c],a=j.charAt(0),j.length>=2&&a==="["?f[c]=h(j):a!=="\\"&&(f[c]=j.replace(/[A-Za-z]/g,function(a){a=a.charCodeAt(0);return"["+String.fromCharCode(a&-33,a|32)+"]"}));return f.join("")}for(var t=0,s=!1,l=!1,p=0,d=a.length;p<d;++p){var g=a[p];if(g.ignoreCase)l=!0;else if(/[a-z]/i.test(g.source.replace(/\\u[\da-f]{4}|\\x[\da-f]{2}|\\[^UXux]/gi,""))){s=!0;l=!1;break}}for(var r=
-{b:8,t:9,n:10,v:11,f:12,r:13},n=[],p=0,d=a.length;p<d;++p){g=a[p];if(g.global||g.multiline)throw Error(""+g);n.push("(?:"+y(g)+")")}return RegExp(n.join("|"),l?"gi":"g")}function M(a){function m(a){switch(a.nodeType){case 1:if(e.test(a.className))break;for(var g=a.firstChild;g;g=g.nextSibling)m(g);g=a.nodeName;if("BR"===g||"LI"===g)h[s]="\n",t[s<<1]=y++,t[s++<<1|1]=a;break;case 3:case 4:g=a.nodeValue,g.length&&(g=p?g.replace(/\r\n?/g,"\n"):g.replace(/[\t\n\r ]+/g," "),h[s]=g,t[s<<1]=y,y+=g.length,
-t[s++<<1|1]=a)}}var e=/(?:^|\s)nocode(?:\s|$)/,h=[],y=0,t=[],s=0,l;a.currentStyle?l=a.currentStyle.whiteSpace:window.getComputedStyle&&(l=document.defaultView.getComputedStyle(a,q).getPropertyValue("white-space"));var p=l&&"pre"===l.substring(0,3);m(a);return{a:h.join("").replace(/\n$/,""),c:t}}function B(a,m,e,h){m&&(a={a:m,d:a},e(a),h.push.apply(h,a.e))}function x(a,m){function e(a){for(var l=a.d,p=[l,"pln"],d=0,g=a.a.match(y)||[],r={},n=0,z=g.length;n<z;++n){var f=g[n],b=r[f],o=void 0,c;if(typeof b===
-"string")c=!1;else{var i=h[f.charAt(0)];if(i)o=f.match(i[1]),b=i[0];else{for(c=0;c<t;++c)if(i=m[c],o=f.match(i[1])){b=i[0];break}o||(b="pln")}if((c=b.length>=5&&"lang-"===b.substring(0,5))&&!(o&&typeof o[1]==="string"))c=!1,b="src";c||(r[f]=b)}i=d;d+=f.length;if(c){c=o[1];var j=f.indexOf(c),k=j+c.length;o[2]&&(k=f.length-o[2].length,j=k-c.length);b=b.substring(5);B(l+i,f.substring(0,j),e,p);B(l+i+j,c,C(b,c),p);B(l+i+k,f.substring(k),e,p)}else p.push(l+i,b)}a.e=p}var h={},y;(function(){for(var e=a.concat(m),
-l=[],p={},d=0,g=e.length;d<g;++d){var r=e[d],n=r[3];if(n)for(var k=n.length;--k>=0;)h[n.charAt(k)]=r;r=r[1];n=""+r;p.hasOwnProperty(n)||(l.push(r),p[n]=q)}l.push(/[\S\s]/);y=L(l)})();var t=m.length;return e}function u(a){var m=[],e=[];a.tripleQuotedStrings?m.push(["str",/^(?:'''(?:[^'\\]|\\[\S\s]|''?(?=[^']))*(?:'''|$)|"""(?:[^"\\]|\\[\S\s]|""?(?=[^"]))*(?:"""|$)|'(?:[^'\\]|\\[\S\s])*(?:'|$)|"(?:[^"\\]|\\[\S\s])*(?:"|$))/,q,"'\""]):a.multiLineStrings?m.push(["str",/^(?:'(?:[^'\\]|\\[\S\s])*(?:'|$)|"(?:[^"\\]|\\[\S\s])*(?:"|$)|`(?:[^\\`]|\\[\S\s])*(?:`|$))/,
-q,"'\"`"]):m.push(["str",/^(?:'(?:[^\n\r'\\]|\\.)*(?:'|$)|"(?:[^\n\r"\\]|\\.)*(?:"|$))/,q,"\"'"]);a.verbatimStrings&&e.push(["str",/^@"(?:[^"]|"")*(?:"|$)/,q]);var h=a.hashComments;h&&(a.cStyleComments?(h>1?m.push(["com",/^#(?:##(?:[^#]|#(?!##))*(?:###|$)|.*)/,q,"#"]):m.push(["com",/^#(?:(?:define|elif|else|endif|error|ifdef|include|ifndef|line|pragma|undef|warning)\b|[^\n\r]*)/,q,"#"]),e.push(["str",/^<(?:(?:(?:\.\.\/)*|\/?)(?:[\w-]+(?:\/[\w-]+)+)?[\w-]+\.h|[a-z]\w*)>/,q])):m.push(["com",/^#[^\n\r]*/,
-q,"#"]));a.cStyleComments&&(e.push(["com",/^\/\/[^\n\r]*/,q]),e.push(["com",/^\/\*[\S\s]*?(?:\*\/|$)/,q]));a.regexLiterals&&e.push(["lang-regex",/^(?:^^\.?|[!+-]|!=|!==|#|%|%=|&|&&|&&=|&=|\(|\*|\*=|\+=|,|-=|->|\/|\/=|:|::|;|<|<<|<<=|<=|=|==|===|>|>=|>>|>>=|>>>|>>>=|[?@[^]|\^=|\^\^|\^\^=|{|\||\|=|\|\||\|\|=|~|break|case|continue|delete|do|else|finally|instanceof|return|throw|try|typeof)\s*(\/(?=[^*/])(?:[^/[\\]|\\[\S\s]|\[(?:[^\\\]]|\\[\S\s])*(?:]|$))+\/)/]);(h=a.types)&&e.push(["typ",h]);a=(""+a.keywords).replace(/^ | $/g,
-"");a.length&&e.push(["kwd",RegExp("^(?:"+a.replace(/[\s,]+/g,"|")+")\\b"),q]);m.push(["pln",/^\s+/,q," \r\n\t\xa0"]);e.push(["lit",/^@[$_a-z][\w$@]*/i,q],["typ",/^(?:[@_]?[A-Z]+[a-z][\w$@]*|\w+_t\b)/,q],["pln",/^[$_a-z][\w$@]*/i,q],["lit",/^(?:0x[\da-f]+|(?:\d(?:_\d+)*\d*(?:\.\d*)?|\.\d\+)(?:e[+-]?\d+)?)[a-z]*/i,q,"0123456789"],["pln",/^\\[\S\s]?/,q],["pun",/^.[^\s\w"-$'./@\\`]*/,q]);return x(m,e)}function D(a,m){function e(a){switch(a.nodeType){case 1:if(k.test(a.className))break;if("BR"===a.nodeName)h(a),
-a.parentNode&&a.parentNode.removeChild(a);else for(a=a.firstChild;a;a=a.nextSibling)e(a);break;case 3:case 4:if(p){var b=a.nodeValue,d=b.match(t);if(d){var c=b.substring(0,d.index);a.nodeValue=c;(b=b.substring(d.index+d[0].length))&&a.parentNode.insertBefore(s.createTextNode(b),a.nextSibling);h(a);c||a.parentNode.removeChild(a)}}}}function h(a){function b(a,d){var e=d?a.cloneNode(!1):a,f=a.parentNode;if(f){var f=b(f,1),g=a.nextSibling;f.appendChild(e);for(var h=g;h;h=g)g=h.nextSibling,f.appendChild(h)}return e}
-for(;!a.nextSibling;)if(a=a.parentNode,!a)return;for(var a=b(a.nextSibling,0),e;(e=a.parentNode)&&e.nodeType===1;)a=e;d.push(a)}var k=/(?:^|\s)nocode(?:\s|$)/,t=/\r\n?|\n/,s=a.ownerDocument,l;a.currentStyle?l=a.currentStyle.whiteSpace:window.getComputedStyle&&(l=s.defaultView.getComputedStyle(a,q).getPropertyValue("white-space"));var p=l&&"pre"===l.substring(0,3);for(l=s.createElement("LI");a.firstChild;)l.appendChild(a.firstChild);for(var d=[l],g=0;g<d.length;++g)e(d[g]);m===(m|0)&&d[0].setAttribute("value",
-m);var r=s.createElement("OL");r.className="linenums";for(var n=Math.max(0,m-1|0)||0,g=0,z=d.length;g<z;++g)l=d[g],l.className="L"+(g+n)%10,l.firstChild||l.appendChild(s.createTextNode("\xa0")),r.appendChild(l);a.appendChild(r)}function k(a,m){for(var e=m.length;--e>=0;){var h=m[e];A.hasOwnProperty(h)?window.console&&console.warn("cannot override language handler %s",h):A[h]=a}}function C(a,m){if(!a||!A.hasOwnProperty(a))a=/^\s*</.test(m)?"default-markup":"default-code";return A[a]}function E(a){var m=
-a.g;try{var e=M(a.h),h=e.a;a.a=h;a.c=e.c;a.d=0;C(m,h)(a);var k=/\bMSIE\b/.test(navigator.userAgent),m=/\n/g,t=a.a,s=t.length,e=0,l=a.c,p=l.length,h=0,d=a.e,g=d.length,a=0;d[g]=s;var r,n;for(n=r=0;n<g;)d[n]!==d[n+2]?(d[r++]=d[n++],d[r++]=d[n++]):n+=2;g=r;for(n=r=0;n<g;){for(var z=d[n],f=d[n+1],b=n+2;b+2<=g&&d[b+1]===f;)b+=2;d[r++]=z;d[r++]=f;n=b}for(d.length=r;h<p;){var o=l[h+2]||s,c=d[a+2]||s,b=Math.min(o,c),i=l[h+1],j;if(i.nodeType!==1&&(j=t.substring(e,b))){k&&(j=j.replace(m,"\r"));i.nodeValue=
-j;var u=i.ownerDocument,v=u.createElement("SPAN");v.className=d[a+1];var x=i.parentNode;x.replaceChild(v,i);v.appendChild(i);e<o&&(l[h+1]=i=u.createTextNode(t.substring(b,o)),x.insertBefore(i,v.nextSibling))}e=b;e>=o&&(h+=2);e>=c&&(a+=2)}}catch(w){"console"in window&&console.log(w&&w.stack?w.stack:w)}}var v=["break,continue,do,else,for,if,return,while"],w=[[v,"auto,case,char,const,default,double,enum,extern,float,goto,int,long,register,short,signed,sizeof,static,struct,switch,typedef,union,unsigned,void,volatile"],
-"catch,class,delete,false,import,new,operator,private,protected,public,this,throw,true,try,typeof"],F=[w,"alignof,align_union,asm,axiom,bool,concept,concept_map,const_cast,constexpr,decltype,dynamic_cast,explicit,export,friend,inline,late_check,mutable,namespace,nullptr,reinterpret_cast,static_assert,static_cast,template,typeid,typename,using,virtual,where"],G=[w,"abstract,boolean,byte,extends,final,finally,implements,import,instanceof,null,native,package,strictfp,super,synchronized,throws,transient"],
-H=[G,"as,base,by,checked,decimal,delegate,descending,dynamic,event,fixed,foreach,from,group,implicit,in,interface,internal,into,is,lock,object,out,override,orderby,params,partial,readonly,ref,sbyte,sealed,stackalloc,string,select,uint,ulong,unchecked,unsafe,ushort,var"],w=[w,"debugger,eval,export,function,get,null,set,undefined,var,with,Infinity,NaN"],I=[v,"and,as,assert,class,def,del,elif,except,exec,finally,from,global,import,in,is,lambda,nonlocal,not,or,pass,print,raise,try,with,yield,False,True,None"],
-J=[v,"alias,and,begin,case,class,def,defined,elsif,end,ensure,false,in,module,next,nil,not,or,redo,rescue,retry,self,super,then,true,undef,unless,until,when,yield,BEGIN,END"],v=[v,"case,done,elif,esac,eval,fi,function,in,local,set,then,until"],K=/^(DIR|FILE|vector|(de|priority_)?queue|list|stack|(const_)?iterator|(multi)?(set|map)|bitset|u?(int|float)\d*)/,N=/\S/,O=u({keywords:[F,H,w,"caller,delete,die,do,dump,elsif,eval,exit,foreach,for,goto,if,import,last,local,my,next,no,our,print,package,redo,require,sub,undef,unless,until,use,wantarray,while,BEGIN,END"+
-I,J,v],hashComments:!0,cStyleComments:!0,multiLineStrings:!0,regexLiterals:!0}),A={};k(O,["default-code"]);k(x([],[["pln",/^[^<?]+/],["dec",/^<!\w[^>]*(?:>|$)/],["com",/^<\!--[\S\s]*?(?:--\>|$)/],["lang-",/^<\?([\S\s]+?)(?:\?>|$)/],["lang-",/^<%([\S\s]+?)(?:%>|$)/],["pun",/^(?:<[%?]|[%?]>)/],["lang-",/^<xmp\b[^>]*>([\S\s]+?)<\/xmp\b[^>]*>/i],["lang-js",/^<script\b[^>]*>([\S\s]*?)(<\/script\b[^>]*>)/i],["lang-css",/^<style\b[^>]*>([\S\s]*?)(<\/style\b[^>]*>)/i],["lang-in.tag",/^(<\/?[a-z][^<>]*>)/i]]),
-["default-markup","htm","html","mxml","xhtml","xml","xsl"]);k(x([["pln",/^\s+/,q," \t\r\n"],["atv",/^(?:"[^"]*"?|'[^']*'?)/,q,"\"'"]],[["tag",/^^<\/?[a-z](?:[\w-.:]*\w)?|\/?>$/i],["atn",/^(?!style[\s=]|on)[a-z](?:[\w:-]*\w)?/i],["lang-uq.val",/^=\s*([^\s"'>]*(?:[^\s"'/>]|\/(?=\s)))/],["pun",/^[/<->]+/],["lang-js",/^on\w+\s*=\s*"([^"]+)"/i],["lang-js",/^on\w+\s*=\s*'([^']+)'/i],["lang-js",/^on\w+\s*=\s*([^\s"'>]+)/i],["lang-css",/^style\s*=\s*"([^"]+)"/i],["lang-css",/^style\s*=\s*'([^']+)'/i],["lang-css",
-/^style\s*=\s*([^\s"'>]+)/i]]),["in.tag"]);k(x([],[["atv",/^[\S\s]+/]]),["uq.val"]);k(u({keywords:F,hashComments:!0,cStyleComments:!0,types:K}),["c","cc","cpp","cxx","cyc","m"]);k(u({keywords:"null,true,false"}),["json"]);k(u({keywords:H,hashComments:!0,cStyleComments:!0,verbatimStrings:!0,types:K}),["cs"]);k(u({keywords:G,cStyleComments:!0}),["java"]);k(u({keywords:v,hashComments:!0,multiLineStrings:!0}),["bsh","csh","sh"]);k(u({keywords:I,hashComments:!0,multiLineStrings:!0,tripleQuotedStrings:!0}),
-["cv","py"]);k(u({keywords:"caller,delete,die,do,dump,elsif,eval,exit,foreach,for,goto,if,import,last,local,my,next,no,our,print,package,redo,require,sub,undef,unless,until,use,wantarray,while,BEGIN,END",hashComments:!0,multiLineStrings:!0,regexLiterals:!0}),["perl","pl","pm"]);k(u({keywords:J,hashComments:!0,multiLineStrings:!0,regexLiterals:!0}),["rb"]);k(u({keywords:w,cStyleComments:!0,regexLiterals:!0}),["js"]);k(u({keywords:"all,and,by,catch,class,else,extends,false,finally,for,if,in,is,isnt,loop,new,no,not,null,of,off,on,or,return,super,then,true,try,unless,until,when,while,yes",
-hashComments:3,cStyleComments:!0,multilineStrings:!0,tripleQuotedStrings:!0,regexLiterals:!0}),["coffee"]);k(x([],[["str",/^[\S\s]+/]]),["regex"]);window.prettyPrintOne=function(a,m,e){var h=document.createElement("PRE");h.innerHTML=a;e&&D(h,e);E({g:m,i:e,h:h});return h.innerHTML};window.prettyPrint=function(a){function m(){for(var e=window.PR_SHOULD_USE_CONTINUATION?l.now()+250:Infinity;p<h.length&&l.now()<e;p++){var n=h[p],k=n.className;if(k.indexOf("prettyprint")>=0){var k=k.match(g),f,b;if(b=
-!k){b=n;for(var o=void 0,c=b.firstChild;c;c=c.nextSibling)var i=c.nodeType,o=i===1?o?b:c:i===3?N.test(c.nodeValue)?b:o:o;b=(f=o===b?void 0:o)&&"CODE"===f.tagName}b&&(k=f.className.match(g));k&&(k=k[1]);b=!1;for(o=n.parentNode;o;o=o.parentNode)if((o.tagName==="pre"||o.tagName==="code"||o.tagName==="xmp")&&o.className&&o.className.indexOf("prettyprint")>=0){b=!0;break}b||((b=(b=n.className.match(/\blinenums\b(?::(\d+))?/))?b[1]&&b[1].length?+b[1]:!0:!1)&&D(n,b),d={g:k,h:n,i:b},E(d))}}p<h.length?setTimeout(m,
-250):a&&a()}for(var e=[document.getElementsByTagName("pre"),document.getElementsByTagName("code"),document.getElementsByTagName("xmp")],h=[],k=0;k<e.length;++k)for(var t=0,s=e[k].length;t<s;++t)h.push(e[k][t]);var e=q,l=Date;l.now||(l={now:function(){return+new Date}});var p=0,d,g=/\blang(?:uage)?-([\w.]+)(?!\S)/;m()};window.PR={createSimpleLexer:x,registerLangHandler:k,sourceDecorator:u,PR_ATTRIB_NAME:"atn",PR_ATTRIB_VALUE:"atv",PR_COMMENT:"com",PR_DECLARATION:"dec",PR_KEYWORD:"kwd",PR_LITERAL:"lit",
-PR_NOCODE:"nocode",PR_PLAIN:"pln",PR_PUNCTUATION:"pun",PR_SOURCE:"src",PR_STRING:"str",PR_TAG:"tag",PR_TYPE:"typ"}})();
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/projectpage.js b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/projectpage.js
index 4536703..21adf81 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/projectpage.js
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/projectpage.js
@@ -12,6 +12,8 @@
   var machineChangeFormToggle = $("#change-machine-toggle");
   var machineNameTitle = $("#project-machine-name");
   var machineChangeCancel = $("#cancel-machine-change");
+  var machineInputForm = $("#machine-input-form");
+  var invalidMachineNameHelp = $("#invalid-machine-name-help");
 
   var freqBuildBtn =  $("#freq-build-btn");
   var freqBuildList = $("#freq-build-list");
@@ -208,8 +210,23 @@
 
 
   /* Change machine functionality */
+  machineChangeInput.keyup(function(){
+    if ($(this).val().indexOf(' ') >= 0) {
+        machineChangeBtn.attr("disabled", "disabled");
+        invalidMachineNameHelp.show();
+        machineInputForm.addClass('has-error');
+    } else {
+        machineChangeBtn.removeAttr("disabled");
+        invalidMachineNameHelp.hide();
+        machineInputForm.removeClass('has-error');
+    }
+  });
 
   machineChangeFormToggle.click(function(){
+    machineChangeInput.val(machineNameTitle.text());
+    machineChangeBtn.removeAttr("disabled");
+    invalidMachineNameHelp.hide();
+    machineInputForm.removeClass('has-error');
     machineForm.slideDown();
     machineNameTitle.hide();
     $(this).hide();
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/table.js b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/table.js
index 176ce57..abcb5ca 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/table.js
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/table.js
@@ -39,6 +39,8 @@
                               ', .show-all-'+ctx.tableName);
 
   function loadData(tableParams){
+    table.trigger("table-loading");
+
     $.ajax({
         type: "GET",
         url: ctx.url,
@@ -200,6 +202,7 @@
     }
 
     /* Add table header and column toggle menu */
+    var column_edit_entries = [];
     for (var i in tableData.columns){
       var col = tableData.columns[i];
       if (col.displayable === false) {
@@ -291,9 +294,17 @@
         defaultHiddenCols.push(col.field_name);
       }
 
-      editColMenu.append(toggler);
+      /* Gather the Edit Column entries */
+      column_edit_entries.push({'title':col.title,'html':toggler});
+
     } /* End for each column */
 
+    /* Append the sorted Edit Column toggler entries */
+    column_edit_entries.sort(function(a,b) {return (a.title > b.title) ? 1 : ((b.title > a.title) ? -1 : 0);} );
+    for (var col in column_edit_entries){
+      editColMenu.append(column_edit_entries[col].html);
+    }
+
     tableChromeDone = true;
   }
 
@@ -835,4 +846,12 @@
 
     $('#filter-modal-'+ctx.tableName).modal('hide');
   });
+
+  table.on("table-loading", function(){
+    table.css("opacity", 0.5);
+  });
+
+  table.on("table-done", function(){
+    table.css("opacity", 1);
+  })
 }
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/tests/test.js b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/tests/test.js
index d7953de..4a4c83f 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/tests/test.js
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/static/js/tests/test.js
@@ -5,6 +5,7 @@
 QUnit.test("Layer alert notification", function(assert) {
   var layer = {
     "layerdetailurl":"/toastergui/project/1/layer/22",
+    "xhrLayerUrl":"/toastergui/xhr_layer/1/9",
     "vcs_url":"git://example.com/example.git",
     "detail":"[ git://example.com/example.git | master ]",
     "vcs_reference":"master",
@@ -15,6 +16,7 @@
   var layerDepsList = [
     {
     "layerdetailurl":"/toastergui/project/1/layer/9",
+    "xhrLayerUrl":"/toastergui/xhr_layer/1/9",
     "vcs_url":"git://example.com/example.git",
     "detail":"[ git://example.com/example.git | master ]",
     "vcs_reference":"master",
@@ -23,6 +25,7 @@
     },
     {
     "layerdetailurl":"/toastergui/project/1/layer/9",
+    "xhrLayerUrl":"/toastergui/xhr_layer/1/9",
     "vcs_url":"git://example.com/example.git",
     "detail":"[ git://example.com/example.git | master ]",
     "vcs_reference":"master",
@@ -66,7 +69,8 @@
 var layer = {
   "id": 1,
   "name":  "meta-testing",
-  "layerdetailurl": "/toastergui/project/1/layer/1"
+  "layerdetailurl": "/toastergui/project/1/layer/1",
+  "xhrLayerUrl": "/toastergui/xhr_layer/1/1"
 };
 
 QUnit.test("Add layer", function(assert){
@@ -144,7 +148,7 @@
 /* Page init functions */
 
 QUnit.test("Import layer page init", function(assert){
-  assert.throws(importLayerPageInit());
+  assert.throws(importLayerPageInit({ xhrGitRevTypeAheadUrl: "url" }));
 });
 
 QUnit.test("Project page init", function(assert){
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/base.html b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/base.html
index 496dd6e..32b4979 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/base.html
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/base.html
@@ -24,7 +24,7 @@
     </script>
     <script src="{% static 'js/jsrender.min.js' %}">
     </script>
-    <script src="{% static 'js/prettify.js' %}">
+    <script src="{% static 'js/highlight.pack.js' %}">
     </script>
     <script src="{% static 'js/libtoaster.js' %}">
     </script>
@@ -100,7 +100,8 @@
         <div class="collapse navbar-collapse" id="global-nav">
             <ul class="nav navbar-nav">
             {% if request.resolver_match.url_name != 'landing' and request.resolver_match.url_name != 'newproject' %}
-              <li {% if request.resolver_match.url_name == 'all-builds' %}
+              <li id="navbar-all-builds"
+              {% if request.resolver_match.url_name == 'all-builds' %}
               class="active"
               {% endif %}>
               <a href="{% url 'all-builds' %}">
@@ -108,7 +109,8 @@
                 All builds
               </a>
               </li>
-              <li {% if request.resolver_match.url_name == 'all-projects'  %}
+              <li id="navbar-all-projects"
+              {% if request.resolver_match.url_name == 'all-projects'  %}
               class="active"
               {% endif %}>
               <a href="{% url 'all-projects' %}">
@@ -117,7 +119,7 @@
               </a>
               </li>
             {% endif %}
-              <li>
+              <li id="navbar-docs">
               <a target="_blank" href="http://www.yoctoproject.org/docs/latest/toaster-manual/toaster-manual.html">
                 <i class="glyphicon glyphicon-book"></i>
                 Documentation
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/basebuildpage.html b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/basebuildpage.html
index f5eee96..a41911f 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/basebuildpage.html
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/basebuildpage.html
@@ -132,16 +132,16 @@
           {% endif %}
           <li class="nav-header">Build</li>
             <li id="menu-configuration"><a href="{% url 'configuration' build.pk %}">Configuration</a></li>
-            <li><a href="{% url 'tasks' build.pk %}">Tasks</a></li>
-            <li><a href="{% url 'recipes' build.pk %}">Recipes</a></li>
-            <li><a href="{% url 'packages' build.pk %}">Packages</a></li>
+            <li id="menu-tasks"><a href="{% url 'tasks' build.pk %}">Tasks</a></li>
+            <li id="menu-recipes"><a href="{% url 'recipes' build.pk %}">Recipes</a></li>
+            <li id="menu-packages"><a href="{% url 'packages' build.pk %}">Packages</a></li>
             <li class="nav-header">Performance</li>
-            <li><a href="{% url 'buildtime' build.pk %}">Time</a></li>
-            <li><a href="{% url 'cputime' build.pk %}">CPU usage</a></li>
-            <li><a href="{% url 'diskio' build.pk %}">Disk I/O</a></li>
+            <li id="menu-time"><a href="{% url 'buildtime' build.pk %}">Time</a></li>
+            <li id="menu-cpu-time"><a href="{% url 'cputime' build.pk %}">CPU usage</a></li>
+            <li id="menu-disk-io"><a href="{% url 'diskio' build.pk %}">Disk I/O</a></li>
 
           <li class="nav-header">Actions</li>
-          <li>
+          <li id="menu-download-build-log">
             <a href="{% url 'build_artifact' build.id 'cookerlog' build.id %}">
               <span class="glyphicon glyphicon-download-alt"></span>
             Download build log
@@ -151,7 +151,7 @@
           {% with build.get_custom_image_recipes as custom_image_recipes %}
             {% if custom_image_recipes.count > 0 %}
               <!-- edit custom image built during this build -->
-              <li>
+              <li id="menu-edit-custom-image">
                 <a href="#" data-role="edit-custom-image-trigger">
                   <span class="glyphicon glyphicon-edit"></span>
                   Edit custom image
@@ -186,7 +186,7 @@
 
             <!-- new custom image from image recipe in this build -->
             {% if build.has_image_recipes %}
-            <li>
+            <li id="menu-new-custom-image">
               <a href="#"  data-role="new-custom-image-trigger">
                 <span class="glyphicon glyphicon-plus"></span>
                 New custom image
@@ -216,7 +216,7 @@
               </script>
             {% endif %}
 
-            <li>
+            <li  id="menu-delete-build">
             <a href="#delete-build-modal" id="delete-build" data-toggle="modal" data-target="#delete-build-modal" class="text-danger">
               <span class="glyphicon glyphicon-trash"></span>
               Delete build
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/importlayer.html b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/importlayer.html
index 1f42696..97d52c7 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/importlayer.html
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/importlayer.html
@@ -13,7 +13,8 @@
   <script>
   $(document).ready(function (){
     var ctx = {
-      xhrImportLayerUrl : "{% url 'xhr_importlayer' %}",
+      xhrLayerUrl : "{% url 'xhr_layer' project.id %}",
+      xhrGitRevTypeAheadUrl : "{% url 'xhr_gitrevtypeahead' %}",
     };
 
     try {
@@ -112,8 +113,10 @@
           Git revision
           <span class="glyphicon glyphicon-question-sign get-help" title="You can provide a Git branch, a tag or a commit SHA as the revision"></span>
         </label>
-        <input type="text" class="form-control" id="layer-git-ref" required>
-        <span class="help-inline" style="display:none;" id="invalid-layer-revision-hint"></span>
+        <span style="display: block">
+        <input type="text" class="form-control" id="layer-git-ref" autocomplete="off" data-minLength="1" data-autocomplete="off" data-provide="typeahead" required>
+        </span>
+        <span class="help-block has-error" style="display:none;" id="invalid-layer-revision-hint">The "HEAD" branch is reserved (only allowed for the "Local Yocto Project" layers)</span>
       </div>
     </fieldset>
 
@@ -123,7 +126,7 @@
         <label for="local-dir-path" class="control-label">Enter the absolute path to the layer directory</label>
         <input type="text" class="form-control" id="local-dir-path" required/>
         <p class="help-block" id="hintError-dir-path-starts-with-slash" style="display:none;">The absolute path must start with "/".</p>
-        <p class="help-block" id="hintError-dir-path" style="display:none;">The directory path cannot include spaces or any of these characters: . \ ? % * : | " " &lt; &gt;</p>
+        <p class="help-block" id="hintError-dir-path" style="display:none;">The directory path cannot include spaces or any of these characters: \ ? % * : | " " &lt; &gt;</p>
       </div>
     </fieldset>
 
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/js-unit-tests.html b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/js-unit-tests.html
index 94ad4f8..ca24896 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/js-unit-tests.html
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/js-unit-tests.html
@@ -14,7 +14,7 @@
 <script src="{% static 'js/bootstrap.min.js' %}"></script>
 <script src="{% static 'js/filtersnippet.js' %}"></script>
 <script src="{% static 'js/importlayer.js' %}"></script>
-<script src="{% static 'js/prettify.js' %}"></script>
+<script src="{% static 'js/highlight.pack.js' %}"></script>
 <script src="{% static 'js/layerBtn.js' %}"></script>
 <script src="{% static 'js/layerDepsModal.js' %}"></script>
 <script src="{% static 'js/projectpage.js' %}"></script>
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/layer_btn.html b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/layer_btn.html
index b2f73eb..1580991 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/layer_btn.html
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/layer_btn.html
@@ -1,4 +1,9 @@
-<a class="btn btn-danger btn-block layer-exists-{{data.pk}} layerbtn"  data-layer='{ "id": {{data.pk}}, "name":  "{{data.layer.name}}", "layerdetailurl": "{%url 'layerdetails' extra.pid data.pk%}"}' data-directive="remove"
+<a class="btn btn-danger btn-block layer-exists-{{data.pk}} layerbtn"  data-layer='{
+   "id": {{data.pk}},
+   "name":  "{{data.layer.name}}",
+   "xhrLayerUrl": "{% url "xhr_layer" extra.pid data.pk %}",
+   "layerdetailurl": "{% url 'layerdetails' extra.pid data.pk %}"
+   }' data-directive="remove"
     {% if data.pk not in extra.current_layers %}
     style="display:none;"
     {% endif %}
@@ -6,7 +11,12 @@
   <span class="glyphicon glyphicon-trash"></span>
   Remove layer
 </a>
-<a class="btn btn-default btn-block layer-add-{{data.pk}} layerbtn" data-layer='{ "id": {{data.pk}}, "name":  "{{data.layer.name}}", "layerdetailurl": "{%url 'layerdetails' extra.pid data.pk%}"}' data-directive="add"
+<a class="btn btn-default btn-block layer-add-{{data.pk}} layerbtn"
+   data-layer='{ "id": {{data.pk}},
+   "name":  "{{data.layer.name}}",
+   "xhrLayerUrl": "{% url "xhr_layer" extra.pid data.pk %}",
+   "layerdetailurl": "{%url "layerdetails" extra.pid data.pk %}"
+   }' data-directive="add"
     {% if data.pk in extra.current_layers %}
     style="display:none;"
     {% endif %}
@@ -14,4 +24,3 @@
   <span class="glyphicon glyphicon-plus"></span>
   Add layer
 </a>
-
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/layerdetails.html b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/layerdetails.html
index f1569bd..e0069db 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/layerdetails.html
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/layerdetails.html
@@ -71,6 +71,7 @@
           inCurrentPrj : false,
           {% endif %}
           layerdetailurl : "{% url 'layerdetails' project.id layerversion.id  %}",
+          xhrLayerUrl: "{% url 'xhr_layer' project.id layerversion.id  %}",
           layer_source: {{layerversion.layer_source|json}},
         },
         layerSourceTypes: {{layer_source|json}},
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/machine_btn.html b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/machine_btn.html
index 5d93d7a..6debe65 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/machine_btn.html
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/machine_btn.html
@@ -4,7 +4,12 @@
     {% endif %}
 >
   Select machine</a>
-<a class="btn btn-default btn-block layerbtn layer-add-{{data.layer_version.id}}" data-layer='{ "id": {{data.layer_version.id}}, "name":  "{{data.layer_version.layer.name}}", "layerdetailurl": "{%url 'layerdetails' extra.pid data.layer_version.id %}"}' data-directive="add"
+<a class="btn btn-default btn-block layerbtn
+layer-add-{{data.layer_version.id}}" data-layer='{ "id":
+   {{data.layer_version.id}}, "name":  "{{data.layer_version.layer.name}}",
+   "layerdetailurl": "{%url 'layerdetails' extra.pid data.layer_version.id %}",
+   "xhrLayerUrl": "{% url "xhr_layer" extra.pid data.layer_version.id %}"
+   }' data-directive="add"
     {% if data.layer_version.pk in extra.current_layers %}
     style="display:none;"
     {% endif %}
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/project.html b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/project.html
index 5abe241..ab7e665 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/project.html
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/project.html
@@ -67,11 +67,12 @@
 
       <form id="select-machine-form" style="display:none;" class="form-inline">
         <span class="help-block">Machine suggestions come from the list of layers added to your project. If you don't see the machine you are looking for, <a href="{% url 'projectmachines' project.id %}">check the full list of machines</a></span>
-        <div class="form-group">
+        <div class="form-group" id="machine-input-form">
           <input class="form-control" id="machine-change-input" autocomplete="off" value="" data-provide="typeahead" data-minlength="1" data-autocomplete="off" type="text">
         </div>
         <button id="machine-change-btn" class="btn btn-default" type="button">Save</button>
         <a href="#" id="cancel-machine-change" class="btn btn-link">Cancel</a>
+        <span class="help-block text-danger" id="invalid-machine-name-help" style="display:none">A valid machine name cannot include spaces.</span>
         <p class="form-link"><a href="{% url 'projectmachines' project.id %}">View compatible machines</a></p>
       </form>
     </div>
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/projectconf.html b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/projectconf.html
index fcf6df2..933c588 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/projectconf.html
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/projectconf.html
@@ -41,7 +41,7 @@
       </div>
       <button id="apply-change-dl_dir" class="btn btn-default" type="button">Save</button>
       <button id="cancel-change-dl_dir" type="button" class="btn btn-link">Cancel</button>
-      <p class="help-block" id="hintError-dl_dir" style="display:none;">The directory path cannot include spaces or any of these characters: . \ ? % * : | " " &lt; &gt;</p>
+      <p class="help-block" id="hintError-dl_dir" style="display:none;">The directory path cannot include spaces or any of these characters: \ ? % * : | " " &lt; &gt;</p>
       <p class="help-block" id="hintError-initialChar-dl_dir" style="display:none;">The directory path should either start with a /, e.g. /home/toaster/downloads; or with a variable, e.g. ${TOPDIR}/downloads.</p>
     </form>
     </dd>
@@ -63,6 +63,7 @@
         <button id="cancel-change-image_fstypes" type="button" class="btn btn-link">Cancel</button>
       </div>
       <p class="help-block text-danger" style="display:none;" id="hintError-image-fs_type">A valid image type cannot include underscores</p>
+      <p class="help-block text-danger" style="display:none;" id="fstypes-error-message">You must select at least one image type</p>
       <label>Or choose from known image types:</label>
       <input id="filter-image_fstypes" type="text" placeholder="Search image types" class="form-control">
       <div id="all-image_fstypes" class="scrolling"></div>
@@ -150,7 +151,7 @@
       </div>
       <button id="apply-change-sstate_dir" class="btn btn-default" type="button">Save</button>
       <button id="cancel-change-sstate_dir" type="button" class="btn btn-link">Cancel</button>
-      <p class="help-block" id="hintError-sstate_dir" style="display:none;">The directory path cannot include spaces or any of these characters: . \ ? % * : | " " &lt; &gt;</p>
+      <p class="help-block" id="hintError-sstate_dir" style="display:none;">The directory path cannot include spaces or any of these characters: \ ? % * : | " " &lt; &gt;</p>
       <p class="help-block" id="hintError-initialChar-sstate_dir" style="display:none;">The directory path should either start with a /, e.g. /home/toaster/sstate-cache; or with a variable, e.g. ${TOPDIR}/sstate-cache.</p>
     </form>
     </dd>
@@ -593,7 +594,7 @@
       var input = $(this);
       var reBeginWithSlash = /^\//;
       var reCheckVariable = /^\$/;
-      var re = /([ <>\\|":\.%\?\*]+)/;
+      var re = /([ <>\\|":%\?\*]+)/;
       var invalidDir = re.test(input.val());
       var invalidSlash = reBeginWithSlash.test(input.val());
       var invalidVar = reCheckVariable.test(input.val());
@@ -716,8 +717,10 @@
     }
     if ($('#new-imagefs_types').val().length === 0) {
       $("#apply-change-image_fstypes").prop("disabled", true);
+      $('#fstypes-error-message').show();
     } else {
       $("#apply-change-image_fstypes").prop("disabled", false);
+      $('#fstypes-error-message').hide();
     }
   });
 
@@ -958,7 +961,7 @@
       var input = $(this);
       var reBeginWithSlash = /^\//;
       var reCheckVariable = /^\$/;
-      var re = /([ <>\\|":\.%\?\*]+)/;
+      var re = /([ <>\\|":%\?\*]+)/;
       var invalidDir = re.test(input.val());
       var invalidSlash = reBeginWithSlash.test(input.val());
       var invalidVar = reCheckVariable.test(input.val());
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/recipe_btn.html b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/recipe_btn.html
index e372964..0ee0ba5 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/recipe_btn.html
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/templates/recipe_btn.html
@@ -5,7 +5,13 @@
  >
   Build recipe
 </a>
-<a class="btn btn-default btn-block layerbtn layer-add-{{data.layer_version.pk}}" data-layer='{ "id": {{data.layer_version.pk}}, "name":  "{{data.layer_version.layer.name}}", "layerdetailurl": "{%url 'layerdetails' extra.pid data.layer_version.pk%}"}' data-directive="add"
+<a class="btn btn-default btn-block layerbtn layer-add-{{data.layer_version.pk}}" 
+  data-layer='{
+  "id": {{data.layer_version.pk}},
+  "name":  "{{data.layer_version.layer.name}}",
+  "layerdetailurl": "{%url "layerdetails" extra.pid data.layer_version.pk%}",
+   "xhrLayerUrl": "{% url "xhr_layer" extra.pid data.layer_version.pk %}"
+   }' data-directive="add"
     {% if data.layer_version.pk in extra.current_layers %}
      style="display:none;"
     {% endif %}
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/typeaheads.py b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/typeaheads.py
index 4ded9ac..58c650f 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/typeaheads.py
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/typeaheads.py
@@ -16,15 +16,17 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
+import subprocess
+
 from toastergui.widgets import ToasterTypeAhead
 from orm.models import Project
 from django.core.urlresolvers import reverse
+from django.core.cache import cache
+
 
 class LayersTypeAhead(ToasterTypeAhead):
     """ Typeahead for layers available and not added in the current project's
     configuration """
-    def __init__(self):
-      super(LayersTypeAhead, self).__init__()
 
     def apply_search(self, search_term, prj, request):
         layers = prj.get_all_compatible_layer_versions()
@@ -33,13 +35,15 @@
         # Unlike the other typeaheads we also don't want to show suggestions
         # for layers already in the project unless required such as when adding
         # layerdeps to a new layer.
-        if ("include_added" in request.GET and
-                request.GET['include_added'] != "true"):
+        if "include_added" in request.GET and \
+           request.GET['include_added'] != "true":
             layers = layers.exclude(
                 pk__in=prj.get_project_layer_versions(pk=True))
 
         primary_results = layers.filter(layer__name__istartswith=search_term)
-        secondary_results = layers.filter(layer__name__icontains=search_term).exclude(pk__in=primary_results)
+        secondary_results = layers.filter(
+            layer__name__icontains=search_term).exclude(
+                pk__in=primary_results)
 
         results = []
 
@@ -49,99 +53,129 @@
             detail = "[ %s | %s ]" % (layer_version.layer.vcs_url,
                                       vcs_reference)
             needed_fields = {
-                'id' : layer_version.pk,
-                'name' : layer_version.layer.name,
-                'layerdetailurl' : layer_version.get_detailspage_url(prj.pk),
-                'vcs_url' : layer_version.layer.vcs_url,
-                'vcs_reference' : vcs_reference,
-                'detail' : detail,
-                'local_source_dir' : layer_version.layer.local_source_dir,
+                'id': layer_version.pk,
+                'name': layer_version.layer.name,
+                'layerdetailurl': layer_version.get_detailspage_url(prj.pk),
+                'xhrLayerUrl': reverse('xhr_layer',
+                                       args=(prj.pk, layer_version.pk)),
+                'vcs_url': layer_version.layer.vcs_url,
+                'vcs_reference': vcs_reference,
+                'detail': detail,
+                'local_source_dir': layer_version.layer.local_source_dir,
             }
 
             results.append(needed_fields)
 
         return results
 
+
 class MachinesTypeAhead(ToasterTypeAhead):
     """ Typeahead for all the machines available in the current project's
     configuration """
-    def __init__(self):
-        super(MachinesTypeAhead, self).__init__()
 
     def apply_search(self, search_term, prj, request):
         machines = prj.get_available_machines()
         machines = machines.order_by("name")
 
         primary_results = machines.filter(name__istartswith=search_term)
-        secondary_results = machines.filter(name__icontains=search_term).exclude(pk__in=primary_results)
-        tertiary_results = machines.filter(layer_version__layer__name__icontains=search_term).exclude(pk__in=primary_results).exclude(pk__in=secondary_results)
+        secondary_results = machines.filter(
+            name__icontains=search_term).exclude(pk__in=primary_results)
+        tertiary_results = machines.filter(
+            layer_version__layer__name__icontains=search_term).exclude(
+                pk__in=primary_results).exclude(pk__in=secondary_results)
 
         results = []
 
-        for machine in list(primary_results) + list(secondary_results) + list(tertiary_results):
+        for machine in list(primary_results) + list(secondary_results) + \
+                list(tertiary_results):
 
             detail = "[ %s ]" % (machine.layer_version.layer.name)
             needed_fields = {
-                'id' : machine.pk,
-                'name' : machine.name,
-                'detail' : detail,
+                'id': machine.pk,
+                'name': machine.name,
+                'detail': detail,
             }
 
             results.append(needed_fields)
-
         return results
 
+
 class RecipesTypeAhead(ToasterTypeAhead):
     """ Typeahead for all the recipes available in the current project's
     configuration """
-    def __init__(self):
-        super(RecipesTypeAhead, self).__init__()
-
     def apply_search(self, search_term, prj, request):
         recipes = prj.get_available_recipes()
         recipes = recipes.order_by("name")
 
-
         primary_results = recipes.filter(name__istartswith=search_term)
-        secondary_results = recipes.filter(name__icontains=search_term).exclude(pk__in=primary_results)
-        tertiary_results = recipes.filter(layer_version__layer__name__icontains=search_term).exclude(pk__in=primary_results).exclude(pk__in=secondary_results)
+        secondary_results = recipes.filter(
+            name__icontains=search_term).exclude(pk__in=primary_results)
+        tertiary_results = recipes.filter(
+            layer_version__layer__name__icontains=search_term).exclude(
+                pk__in=primary_results).exclude(pk__in=secondary_results)
 
         results = []
 
-        for recipe in list(primary_results) + list(secondary_results) + list(tertiary_results):
+        for recipe in list(primary_results) + list(secondary_results) + \
+                list(tertiary_results):
 
             detail = "[ %s ]" % (recipe.layer_version.layer.name)
             needed_fields = {
-                'id' : recipe.pk,
-                'name' : recipe.name,
-                'detail' : detail,
+                'id': recipe.pk,
+                'name': recipe.name,
+                'detail': detail,
             }
 
             results.append(needed_fields)
 
         return results
 
+
 class ProjectsTypeAhead(ToasterTypeAhead):
     """ Typeahead for all the projects, except for command line builds """
-    def __init__(self):
-        super(ProjectsTypeAhead, self).__init__()
-
     def apply_search(self, search_term, prj, request):
         projects = Project.objects.exclude(is_default=True).order_by("name")
 
         primary_results = projects.filter(name__istartswith=search_term)
-        secondary_results = projects.filter(name__icontains=search_term).exclude(pk__in=primary_results)
+        secondary_results = projects.filter(
+            name__icontains=search_term).exclude(pk__in=primary_results)
 
         results = []
 
         for project in list(primary_results) + list(secondary_results):
             needed_fields = {
-                'id' : project.pk,
-                'name' : project.name,
-                'detail' : "",
-                'projectPageUrl' : reverse('project', args=(project.pk,))
+                'id': project.pk,
+                'name': project.name,
+                'detail': "",
+                'projectPageUrl': reverse('project', args=(project.pk,))
             }
 
             results.append(needed_fields)
 
         return results
+
+
+class GitRevisionTypeAhead(ToasterTypeAhead):
+    def apply_search(self, search_term, prj, request):
+        results = []
+        git_url = request.GET.get('git_url')
+        ls_remote = cache.get(git_url)
+
+        if ls_remote is None:
+            ls_remote = subprocess.check_output(['git', 'ls-remote', git_url],
+                                                universal_newlines=True)
+            ls_remote = ls_remote.splitlines()
+            # Avoid fetching the list of git refs on each new input
+            cache.set(git_url, ls_remote, 120)
+
+        for rev in ls_remote:
+            git_rev = str(rev).split("/")[-1:][0]
+            # "HEAD" has a special meaning in Toaster...  YOCTO #9924
+            if "HEAD" in git_rev:
+                continue
+
+            if git_rev.startswith(search_term):
+                results.append({'name': git_rev,
+                                'detail': '[ %s ]' % str(rev)})
+
+        return results
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/urls.py b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/urls.py
index ece9ac1..d92f190 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/urls.py
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/urls.py
@@ -182,20 +182,23 @@
             typeaheads.RecipesTypeAhead.as_view(), name='xhr_recipestypeahead'),
         url(r'^xhr_typeahead/projects$',
             typeaheads.ProjectsTypeAhead.as_view(), name='xhr_projectstypeahead'),
-
-
+        url(r'^xhr_typeahead/gitrev$',
+            typeaheads.GitRevisionTypeAhead.as_view(),
+            name='xhr_gitrevtypeahead'),
 
         url(r'^xhr_testreleasechange/(?P<pid>\d+)$', 'xhr_testreleasechange',
             name='xhr_testreleasechange'),
         url(r'^xhr_configvaredit/(?P<pid>\d+)$', 'xhr_configvaredit',
             name='xhr_configvaredit'),
 
-        url(r'^xhr_importlayer/$', 'xhr_importlayer', name='xhr_importlayer'),
-
         url(r'^xhr_layer/(?P<pid>\d+)/(?P<layerversion_id>\d+)$',
             api.XhrLayer.as_view(),
             name='xhr_layer'),
 
+        url(r'^xhr_layer/(?P<pid>\d+)$',
+            api.XhrLayer.as_view(),
+            name='xhr_layer'),
+
         # JS Unit tests
         url(r'^js-unit-tests/$', 'jsunittests', name='js-unit-tests'),
 
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/views.py b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/views.py
index 2efb0fd..75c5911 100755
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/views.py
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/views.py
@@ -120,32 +120,6 @@
         raise TypeError("Unserializable object %s (%s) of type %s" % ( obj, dir(obj), type(obj)))
 
 
-def _template_renderer(template):
-    def func_wrapper(view):
-        def returned_wrapper(request, *args, **kwargs):
-            try:
-                context = view(request, *args, **kwargs)
-            except RedirectException as e:
-                return e.get_redirect_response()
-
-            if request.GET.get('format', None) == 'json':
-                # objects is a special keyword - it's a Page, but we need the actual objects here
-                # in XHR, the objects come in the "rows" property
-                if "objects" in context:
-                    context["rows"] = context["objects"].object_list
-                    del context["objects"]
-
-                # we're about to return; to keep up with the XHR API, we set the error to OK
-                context["error"] = "ok"
-
-                return HttpResponse(jsonfilter(context, default=objtojson ),
-                            content_type = "application/json; charset=utf-8")
-            else:
-                return render(request, template, context)
-        return returned_wrapper
-    return func_wrapper
-
-
 def _lv_to_dict(prj, x = None):
     if x is None:
         def wrapper(x):
@@ -1509,121 +1483,6 @@
             return HttpResponse(json.dumps({"error":str(e) + "\n" + traceback.format_exc()}), content_type = "application/json")
 
 
-    def xhr_importlayer(request):
-        if ('vcs_url' not in request.POST or
-            'name' not in request.POST or
-            'git_ref' not in request.POST or
-            'project_id' not in request.POST):
-          return HttpResponse(jsonfilter({"error": "Missing parameters; requires vcs_url, name, git_ref and project_id"}), content_type = "application/json")
-
-        layers_added = [];
-
-        # Rudimentary check for any possible html tags
-        for val in request.POST.values():
-            if "<" in val:
-                return HttpResponse(jsonfilter(
-                    {"error": "Invalid character <"}),
-                    content_type="application/json")
-
-        prj = Project.objects.get(pk=request.POST['project_id'])
-
-        # Strip trailing/leading whitespace from all values
-        # put into a new dict because POST one is immutable.
-        post_data = dict()
-        for key,val in request.POST.items():
-          post_data[key] = val.strip()
-
-
-        try:
-            layer, layer_created = Layer.objects.get_or_create(name=post_data['name'])
-        except MultipleObjectsReturned:
-            return HttpResponse(jsonfilter({"error": "hint-layer-exists"}), content_type = "application/json")
-
-        if layer:
-            if layer_created:
-                layer.vcs_url = post_data.get('vcs_url')
-                layer.local_source_dir = post_data.get('local_source_dir')
-                layer.up_date = timezone.now()
-                layer.save()
-            else:
-                # We have an existing layer by this name, let's see if the git
-                # url is the same, if it is then we can just create a new layer
-                # version for this layer. Otherwise we need to bail out.
-                if layer.vcs_url != post_data['vcs_url']:
-                    return HttpResponse(jsonfilter({"error": "hint-layer-exists-with-different-url" , "current_url" : layer.vcs_url, "current_id": layer.id }), content_type = "application/json")
-
-            layer_version, version_created = \
-                Layer_Version.objects.get_or_create(
-                        layer_source=LayerSource.TYPE_IMPORTED,
-                        layer=layer, project=prj,
-                        release=prj.release,
-                        branch=post_data['git_ref'],
-                        commit=post_data['git_ref'],
-                        dirpath=post_data['dir_path'])
-
-            if layer_version:
-                if not version_created:
-                    return HttpResponse(jsonfilter({"error":
-                                                    "hint-layer-version-exists",
-                                                    "existing_layer_version":
-                                                    layer_version.id }),
-                                        content_type = "application/json")
-
-                layer_version.layer_source = LayerSource.TYPE_IMPORTED
-
-                layer_version.up_date = timezone.now()
-                layer_version.save()
-
-                # Add the dependencies specified for this new layer
-                if ('layer_deps' in post_data and
-                    version_created and
-                    len(post_data["layer_deps"]) > 0):
-                    for layer_dep_id in post_data["layer_deps"].split(","):
-
-                        layer_dep_obj = Layer_Version.objects.get(pk=layer_dep_id)
-                        LayerVersionDependency.objects.get_or_create(layer_version=layer_version, depends_on=layer_dep_obj)
-                        # Now add them to the project, we could get an execption
-                        # if the project now contains the exact
-                        # dependency already (like modified on another page)
-                        try:
-                            prj_layer, prj_layer_created = ProjectLayer.objects.get_or_create(layercommit=layer_dep_obj, project=prj)
-                        except IntegrityError as e:
-                            logger.warning("Integrity error while saving Project Layers: %s (original %s)" % (e, e.__cause__))
-                            continue
-
-                        if prj_layer_created:
-                            layerdepdetailurl = reverse('layerdetails', args=(prj.id, layer_dep_obj.pk))
-                            layers_added.append({'id': layer_dep_obj.id, 'name': Layer.objects.get(id=layer_dep_obj.layer_id).name, 'layerdetailurl': layerdepdetailurl })
-
-
-                # If an old layer version exists in our project then remove it
-                for prj_layers in ProjectLayer.objects.filter(project=prj):
-                    dup_layer_v = Layer_Version.objects.filter(id=prj_layers.layercommit_id, layer_id=layer.id)
-                    if len(dup_layer_v) >0 :
-                        prj_layers.delete()
-
-                # finally add the imported layer (version id) to the project
-                ProjectLayer.objects.create(layercommit=layer_version, project=prj,optional=1)
-
-            else:
-                # We didn't create a layer version so back out now and clean up.
-                if layer_created:
-                    layer.delete()
-
-                return HttpResponse(jsonfilter({"error": "Uncaught error: Could not create layer version"}), content_type = "application/json")
-
-        layerdetailurl = reverse('layerdetails', args=(prj.id, layer_version.pk))
-
-        json_response = {"error": "ok",
-                         "imported_layer" : {
-                           "name" : layer.name,
-                           "id": layer_version.id,
-                           "layerdetailurl": layerdetailurl,
-                         },
-                         "deps_added": layers_added }
-
-        return HttpResponse(jsonfilter(json_response), content_type = "application/json")
-
     def customrecipe_download(request, pid, recipe_id):
         recipe = get_object_or_404(CustomImageRecipe, pk=recipe_id)
 
@@ -1643,8 +1502,6 @@
         }
         return render(request, template, context)
 
-    # TODO merge with api pseudo api here is used for deps modal
-    @_template_renderer('layerdetails.html')
     def layerdetails(request, pid, layerid):
         project = Project.objects.get(pk=pid)
         layer_version = Layer_Version.objects.get(pk=layerid)
@@ -1672,7 +1529,7 @@
             'projectlayers': list(project_layers)
         }
 
-        return context
+        return render(request, 'layerdetails.html', context)
 
 
     def get_project_configvars_context():
@@ -1692,7 +1549,6 @@
 
         return(vars_managed,sorted(vars_fstypes),vars_blacklist)
 
-    @_template_renderer("projectconf.html")
     def projectconf(request, pid):
 
         try:
@@ -1763,7 +1619,7 @@
         except (ProjectVariable.DoesNotExist, BuildEnvironment.DoesNotExist):
             pass
 
-        return context
+        return render(request, "projectconf.html", context)
 
     def _file_names_for_artifact(build, artifact_type, artifact_id):
         """
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/widgets.py b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/widgets.py
index 026903d..6b7b981 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/widgets.py
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastergui/widgets.py
@@ -42,6 +42,8 @@
 import collections
 import re
 
+from toastergui.tablefilter import TableFilterMap
+
 try:
     from urllib import unquote_plus
 except ImportError:
@@ -50,8 +52,6 @@
 import logging
 logger = logging.getLogger("toaster")
 
-from toastergui.tablefilter import TableFilterMap
-
 
 class NoFieldOrDataName(Exception):
     pass
@@ -259,9 +259,9 @@
                     queries = query
 
             if search_queries:
-               search_queries &= queries
+                search_queries &= queries
             else:
-               search_queries = queries
+                search_queries = queries
 
         self.queryset = self.queryset.filter(search_queries)
 
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastermain/settings.py b/import-layers/yocto-poky/bitbake/lib/toaster/toastermain/settings.py
index aec9dbb..1fd649c 100644
--- a/import-layers/yocto-poky/bitbake/lib/toaster/toastermain/settings.py
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastermain/settings.py
@@ -38,8 +38,7 @@
 
 MANAGERS = ADMINS
 
-TOASTER_SQLITE_DEFAULT_DIR = os.path.join(os.environ.get('TOASTER_DIR', ''),
-                                          'build')
+TOASTER_SQLITE_DEFAULT_DIR = os.environ.get('TOASTER_DIR')
 
 DATABASES = {
     'default': {
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/toastermain/settings_production_example.py b/import-layers/yocto-poky/bitbake/lib/toaster/toastermain/settings_production_example.py
new file mode 100644
index 0000000..61a2888
--- /dev/null
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/toastermain/settings_production_example.py
@@ -0,0 +1,58 @@
+#
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2016        Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+# See Django documentation for more information about deployment
+# https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/
+
+# Toaster production settings example overlay
+# To use this copy this example to "settings_production.py" and set in your
+# environment DJANGO_SETTINGS_MODULE=toastermain.settings_production
+# This can be permanently set in a new .wsgi file
+
+from toastermain.settings import *  # NOQA
+
+# Set this value!
+SECRET_KEY = None
+
+# Switch off any debugging
+DEBUG = True
+TEMPLATE_DEBUG = DEBUG
+
+DATABASES = {
+     'default': {
+         'ENGINE': 'django.db.backends.mysql',
+         'NAME': 'toaster_data',
+         'USER': 'toaster',
+         'PASSWORD': 'yourpasswordhere',
+         'HOST': '127.0.0.1',
+         'PORT': '3306',
+     }
+}
+
+# Location where static files will be placed by "manage.py collectstatic"
+STATIC_ROOT = '/var/www/static-toaster/'
+
+# URL prefix for static files.
+STATIC_URL = '/static-toaster/'
+
+# Hosts that Django will serve
+# https://docs.djangoproject.com/en/1.8/ref/settings/#std:setting-ALLOWED_HOSTS
+ALLOWED_HOSTS = ['toaster-example.example.com']