diff --git a/yocto-poky/bitbake/lib/bb/fetch2/__init__.py b/yocto-poky/bitbake/lib/bb/fetch2/__init__.py
index a9c044b..1fa6702 100644
--- a/yocto-poky/bitbake/lib/bb/fetch2/__init__.py
+++ b/yocto-poky/bitbake/lib/bb/fetch2/__init__.py
@@ -29,11 +29,9 @@
 from __future__ import print_function
 import os, re
 import signal
-import glob
 import logging
 import urllib
 import urlparse
-import operator
 import bb.persist_data, bb.utils
 import bb.checksum
 from bb import data
@@ -329,7 +327,7 @@
     def path(self, path):
         self._path = path
 
-        if re.compile("^/").match(path):
+        if not path or re.compile("^/").match(path):
             self.relative = False
         else:
             self.relative = True
@@ -377,9 +375,12 @@
     if locidx != -1 and type.lower() != 'file':
         host = location[:locidx]
         path = location[locidx:]
-    else:
+    elif type.lower() == 'file':
         host = ""
         path = location
+    else:
+        host = location
+        path = ""
     if user:
         m = re.compile('(?P<user>[^:]+)(:?(?P<pswd>.*))').match(user)
         if m:
@@ -515,13 +516,13 @@
         if hasattr(m, "init"):
             m.init(d)
 
-def fetcher_parse_save(d):
-    _checksum_cache.save_extras(d)
+def fetcher_parse_save():
+    _checksum_cache.save_extras()
 
-def fetcher_parse_done(d):
-    _checksum_cache.save_merge(d)
+def fetcher_parse_done():
+    _checksum_cache.save_merge()
 
-def fetcher_compare_revisions(d):
+def fetcher_compare_revisions():
     """
     Compare the revisions in the persistant cache with current values and
     return true/false on whether they've changed.
@@ -574,10 +575,10 @@
     else:
         sha256data = bb.utils.sha256_file(ud.localpath)
 
-    if ud.method.recommends_checksum(ud):
+    if ud.method.recommends_checksum(ud) and not ud.md5_expected and not ud.sha256_expected:
         # If strict checking enabled and neither sum defined, raise error
         strict = d.getVar("BB_STRICT_CHECKSUM", True) or "0"
-        if (strict == "1") and not (ud.md5_expected or ud.sha256_expected):
+        if strict == "1":
             logger.error('No checksum specified for %s, please add at least one to the recipe:\n'
                              'SRC_URI[%s] = "%s"\nSRC_URI[%s] = "%s"' %
                              (ud.localpath, ud.md5_name, md5data,
@@ -585,34 +586,22 @@
             raise NoChecksumError('Missing SRC_URI checksum', ud.url)
 
         # Log missing sums so user can more easily add them
-        if not ud.md5_expected:
-            logger.warn('Missing md5 SRC_URI checksum for %s, consider adding to the recipe:\n'
-                        'SRC_URI[%s] = "%s"',
-                        ud.localpath, ud.md5_name, md5data)
-
-        if not ud.sha256_expected:
-            logger.warn('Missing sha256 SRC_URI checksum for %s, consider adding to the recipe:\n'
-                        'SRC_URI[%s] = "%s"',
-                        ud.localpath, ud.sha256_name, sha256data)
-
-    md5mismatch = False
-    sha256mismatch = False
-
-    if ud.md5_expected != md5data:
-        md5mismatch = True
-
-    if ud.sha256_expected != sha256data:
-        sha256mismatch = True
+        logger.warn('Missing md5 SRC_URI checksum for %s, consider adding to the recipe:\n'
+                    'SRC_URI[%s] = "%s"',
+                    ud.localpath, ud.md5_name, md5data)
+        logger.warn('Missing sha256 SRC_URI checksum for %s, consider adding to the recipe:\n'
+                    'SRC_URI[%s] = "%s"',
+                    ud.localpath, ud.sha256_name, sha256data)
 
     # We want to alert the user if a checksum is defined in the recipe but
     # it does not match.
     msg = ""
     mismatch = False
-    if md5mismatch and ud.md5_expected:
+    if ud.md5_expected and ud.md5_expected != md5data:
         msg = msg + "\nFile: '%s' has %s checksum %s when %s was expected" % (ud.localpath, 'md5', md5data, ud.md5_expected)
         mismatch = True;
 
-    if sha256mismatch and ud.sha256_expected:
+    if ud.sha256_expected and ud.sha256_expected != sha256data:
         msg = msg + "\nFile: '%s' has %s checksum %s when %s was expected" % (ud.localpath, 'sha256', sha256data, ud.sha256_expected)
         mismatch = True;
 
@@ -637,6 +626,9 @@
     Returns True, if the donestamp exists and is valid, False otherwise. When
     returning False, any existing done stamps are removed.
     """
+    if not ud.needdonestamp:
+        return True
+
     if not os.path.exists(ud.donestamp):
         return False
 
@@ -685,7 +677,8 @@
         # incorrect stamp file.
         logger.warn("Checksum mismatch for local file %s\n"
                     "Cleaning and trying again." % ud.localpath)
-        rename_bad_checksum(ud, e.checksum)
+        if os.path.exists(ud.localpath):
+            rename_bad_checksum(ud, e.checksum)
         bb.utils.remove(ud.donestamp)
     return False
 
@@ -695,6 +688,9 @@
         donestamp is file stamp indicating the whole fetching is done
         this function update the stamp after verifying the checksum
     """
+    if not ud.needdonestamp:
+        return
+
     if os.path.exists(ud.donestamp):
         # Touch the done stamp file to show active use of the download
         try:
@@ -703,11 +699,21 @@
             # Errors aren't fatal here
             pass
     else:
-        checksums = verify_checksum(ud, d)
-        # Store the checksums for later re-verification against the recipe
-        with open(ud.donestamp, "wb") as cachefile:
-            p = pickle.Pickler(cachefile, pickle.HIGHEST_PROTOCOL)
-            p.dump(checksums)
+        try:
+            checksums = verify_checksum(ud, d)
+            # Store the checksums for later re-verification against the recipe
+            with open(ud.donestamp, "wb") as cachefile:
+                p = pickle.Pickler(cachefile, pickle.HIGHEST_PROTOCOL)
+                p.dump(checksums)
+        except ChecksumError as e:
+            # Checksums failed to verify, trigger re-download and remove the
+            # incorrect stamp file.
+            logger.warn("Checksum mismatch for local file %s\n"
+                        "Cleaning and trying again." % ud.localpath)
+            if os.path.exists(ud.localpath):
+                rename_bad_checksum(ud, e.checksum)
+            bb.utils.remove(ud.donestamp)
+            raise
 
 def subprocess_setup():
     # Python installs a SIGPIPE handler by default. This is usually not what
@@ -718,7 +724,7 @@
 def get_autorev(d):
     #  only not cache src rev in autorev case
     if d.getVar('BB_SRCREV_POLICY', True) != "cache":
-        d.setVar('__BB_DONT_CACHE', '1')
+        d.setVar('BB_DONT_CACHE', '1')
     return "AUTOINC"
 
 def get_srcrev(d, method_name='sortable_revision'):
@@ -920,6 +926,10 @@
 def try_mirror_url(fetch, origud, ud, ld, check = False):
     # Return of None or a value means we're finished
     # False means try another url
+
+    if ud.lockfile and ud.lockfile != origud.lockfile:
+        lf = bb.utils.lockfile(ud.lockfile)
+
     try:
         if check:
             found = ud.method.checkstatus(fetch, ud, ld)
@@ -946,8 +956,9 @@
         if origud.mirrortarball and os.path.basename(ud.localpath) == os.path.basename(origud.mirrortarball) \
                 and os.path.basename(ud.localpath) != os.path.basename(origud.localpath):
             # Create donestamp in old format to avoid triggering a re-download
-            bb.utils.mkdirhier(os.path.dirname(ud.donestamp))
-            open(ud.donestamp, 'w').close()
+            if ud.donestamp:
+                bb.utils.mkdirhier(os.path.dirname(ud.donestamp))
+                open(ud.donestamp, 'w').close()
             dest = os.path.join(dldir, os.path.basename(ud.localpath))
             if not os.path.exists(dest):
                 os.symlink(ud.localpath, dest)
@@ -973,7 +984,8 @@
         if isinstance(e, ChecksumError):
             logger.warn("Mirror checksum failure for url %s (original url: %s)\nCleaning and trying again." % (ud.url, origud.url))
             logger.warn(str(e))
-            rename_bad_checksum(ud, e.checksum)
+            if os.path.exists(ud.localpath):
+                rename_bad_checksum(ud, e.checksum)
         elif isinstance(e, NoChecksumError):
             raise
         else:
@@ -984,6 +996,10 @@
         except UnboundLocalError:
             pass
         return False
+    finally:
+        if ud.lockfile and ud.lockfile != origud.lockfile:
+            bb.utils.unlockfile(lf)
+
 
 def try_mirrors(fetch, d, origud, mirrors, check = False):
     """
@@ -1014,7 +1030,7 @@
         return True
 
     pkgname = d.expand(d.getVar('PN', False))
-    trusted_hosts = d.getVarFlag('BB_ALLOWED_NETWORKS', pkgname)
+    trusted_hosts = d.getVarFlag('BB_ALLOWED_NETWORKS', pkgname, False)
 
     if not trusted_hosts:
         trusted_hosts = d.getVar('BB_ALLOWED_NETWORKS', True)
@@ -1028,6 +1044,7 @@
     if not network:
         return True
 
+    network = network.split(':')[0]
     network = network.lower()
 
     for host in trusted_hosts.split(" "):
@@ -1120,48 +1137,7 @@
     it proceeds
 
     """
-
-    def checksum_file(f):
-        try:
-            checksum = _checksum_cache.get_checksum(f)
-        except OSError as e:
-            bb.warn("Unable to get checksum for %s SRC_URI entry %s: %s" % (pn, os.path.basename(f), e))
-            return None
-        return checksum
-
-    def checksum_dir(pth):
-        # Handle directories recursively
-        dirchecksums = []
-        for root, dirs, files in os.walk(pth):
-            for name in files:
-                fullpth = os.path.join(root, name)
-                checksum = checksum_file(fullpth)
-                if checksum:
-                    dirchecksums.append((fullpth, checksum))
-        return dirchecksums
-
-    checksums = []
-    for pth in filelist.split():
-        exist = pth.split(":")[1]
-        if exist == "False":
-            continue
-        pth = pth.split(":")[0]
-        if '*' in pth:
-            # Handle globs
-            for f in glob.glob(pth):
-                if os.path.isdir(f):
-                    checksums.extend(checksum_dir(f))
-                else:
-                    checksum = checksum_file(f)
-                    checksums.append((f, checksum))
-        elif os.path.isdir(pth):
-            checksums.extend(checksum_dir(pth))
-        else:
-            checksum = checksum_file(pth)
-            checksums.append((pth, checksum))
-
-    checksums.sort(key=operator.itemgetter(1))
-    return checksums
+    return _checksum_cache.get_checksums(filelist, pn)
 
 
 class FetchData(object):
@@ -1171,6 +1147,7 @@
     def __init__(self, url, d, localonly = False):
         # localpath is the location of a downloaded result. If not set, the file is local.
         self.donestamp = None
+        self.needdonestamp = True
         self.localfile = ""
         self.localpath = None
         self.lockfile = None
@@ -1197,13 +1174,13 @@
         elif self.type not in ["http", "https", "ftp", "ftps", "sftp"]:
             self.md5_expected = None
         else:
-            self.md5_expected = d.getVarFlag("SRC_URI", self.md5_name)
+            self.md5_expected = d.getVarFlag("SRC_URI", self.md5_name, True)
         if self.sha256_name in self.parm:
             self.sha256_expected = self.parm[self.sha256_name]
         elif self.type not in ["http", "https", "ftp", "ftps", "sftp"]:
             self.sha256_expected = None
         else:
-            self.sha256_expected = d.getVarFlag("SRC_URI", self.sha256_name)
+            self.sha256_expected = d.getVarFlag("SRC_URI", self.sha256_name, True)
         self.ignore_checksums = False
 
         self.names = self.parm.get("name",'default').split(',')
@@ -1235,13 +1212,20 @@
             self.localpath = self.method.localpath(self, d)
 
         dldir = d.getVar("DL_DIR", True)
+
+        if not self.needdonestamp:
+            return
+
         # Note: .done and .lock files should always be in DL_DIR whereas localpath may not be.
         if self.localpath and self.localpath.startswith(dldir):
             basepath = self.localpath
         elif self.localpath:
             basepath = dldir + os.sep + os.path.basename(self.localpath)
-        else:
+        elif self.basepath or self.basename:
             basepath = dldir + os.sep + (self.basepath or self.basename)
+        else:
+             bb.fatal("Can't determine lock path for url %s" % url)
+
         self.donestamp = basepath + '.done'
         self.lockfile = basepath + '.lock'
 
@@ -1355,6 +1339,11 @@
         iterate = False
         file = urldata.localpath
 
+        # Localpath can't deal with 'dir/*' entries, so it converts them to '.',
+        # but it must be corrected back for local files copying
+        if urldata.basename == '*' and file.endswith('/.'):
+            file = '%s/%s' % (file.rstrip('/.'), urldata.path)
+
         try:
             unpack = bb.utils.to_boolean(urldata.parm.get('unpack'), True)
         except ValueError as exc:
@@ -1407,51 +1396,40 @@
                     cmd = 'rpm2cpio.sh %s | cpio -id' % (file)
             elif file.endswith('.deb') or file.endswith('.ipk'):
                 cmd = 'ar -p %s data.tar.gz | zcat | tar --no-same-owner -xpf -' % file
+            elif file.endswith('.tar.7z'):
+                cmd = '7z x -so %s | tar xf - ' % file
+            elif file.endswith('.7z'):
+                cmd = '7za x -y %s 1>/dev/null' % file
+
+        # If 'subdir' param exists, create a dir and use it as destination for unpack cmd
+        if 'subdir' in urldata.parm:
+            unpackdir = '%s/%s' % (rootdir, urldata.parm.get('subdir'))
+            bb.utils.mkdirhier(unpackdir)
+        else:
+            unpackdir = rootdir
 
         if not unpack or not cmd:
             # If file == dest, then avoid any copies, as we already put the file into dest!
-            dest = os.path.join(rootdir, os.path.basename(file))
-            if (file != dest) and not (os.path.exists(dest) and os.path.samefile(file, dest)):
-                if os.path.isdir(file):
-                    # If for example we're asked to copy file://foo/bar, we need to unpack the result into foo/bar
-                    basepath = getattr(urldata, "basepath", None)
-                    destdir = "."
-                    if basepath and basepath.endswith("/"):
-                        basepath = basepath.rstrip("/")
-                    elif basepath:
-                        basepath = os.path.dirname(basepath)
-                    if basepath and basepath.find("/") != -1:
-                        destdir = basepath[:basepath.rfind('/')]
-                        destdir = destdir.strip('/')
-                    if destdir != "." and not os.access("%s/%s" % (rootdir, destdir), os.F_OK):
-                        os.makedirs("%s/%s" % (rootdir, destdir))
-                    cmd = 'cp -fpPR %s %s/%s/' % (file, rootdir, destdir)
-                    #cmd = 'tar -cf - -C "%d" -ps . | tar -xf - -C "%s/%s/"' % (file, rootdir, destdir)
-                else:
-                    # The "destdir" handling was specifically done for FILESPATH
-                    # items.  So, only do so for file:// entries.
-                    if urldata.type == "file" and urldata.path.find("/") != -1:
-                       destdir = urldata.path.rsplit("/", 1)[0]
-                       if urldata.parm.get('subdir') != None:
-                          destdir = urldata.parm.get('subdir') + "/" + destdir
-                    else:
-                       if urldata.parm.get('subdir') != None:
-                          destdir = urldata.parm.get('subdir')
-                       else:
-                          destdir = "."
-                    bb.utils.mkdirhier("%s/%s" % (rootdir, destdir))
-                    cmd = 'cp -f %s %s/%s/' % (file, rootdir, destdir)
+            dest = os.path.join(unpackdir, os.path.basename(file))
+            if file != dest and not (os.path.exists(dest) and os.path.samefile(file, dest)):
+                destdir = '.'
+                # For file:// entries all intermediate dirs in path must be created at destination
+                if urldata.type == "file":
+                    # Trailing '/' does a copying to wrong place
+                    urlpath = urldata.path.rstrip('/')
+                    # Want files places relative to cwd so no leading '/'
+                    urlpath = urlpath.lstrip('/')
+                    if urlpath.find("/") != -1:
+                        destdir = urlpath.rsplit("/", 1)[0] + '/'
+                        bb.utils.mkdirhier("%s/%s" % (unpackdir, destdir))
+                cmd = 'cp -fpPR %s %s' % (file, destdir)
 
         if not cmd:
             return
 
-        # Change to subdir before executing command
+        # Change to unpackdir before executing command
         save_cwd = os.getcwd();
-        os.chdir(rootdir)
-        if 'subdir' in urldata.parm:
-            newdir = ("%s/%s" % (rootdir, urldata.parm.get('subdir')))
-            bb.utils.mkdirhier(newdir)
-            os.chdir(newdir)
+        os.chdir(unpackdir)
 
         path = data.getVar('PATH', True)
         if path:
@@ -1578,7 +1556,8 @@
             m = ud.method
             localpath = ""
 
-            lf = bb.utils.lockfile(ud.lockfile)
+            if ud.lockfile:
+                lf = bb.utils.lockfile(ud.lockfile)
 
             try:
                 self.d.setVar("BB_NO_NETWORK", network)
@@ -1617,7 +1596,8 @@
                         if isinstance(e, ChecksumError):
                             logger.warn("Checksum failure encountered with download of %s - will attempt other sources if available" % u)
                             logger.debug(1, str(e))
-                            rename_bad_checksum(ud, e.checksum)
+                            if os.path.exists(ud.localpath):
+                                rename_bad_checksum(ud, e.checksum)
                         elif isinstance(e, NoChecksumError):
                             raise
                         else:
@@ -1644,7 +1624,8 @@
                 raise
 
             finally:
-                bb.utils.unlockfile(lf)
+                if ud.lockfile:
+                    bb.utils.unlockfile(lf)
 
     def checkstatus(self, urls=None):
         """
@@ -1686,9 +1667,6 @@
             ud = self.ud[u]
             ud.setup_localpath(self.d)
 
-            if self.d.expand(self.localpath) is None:
-                continue
-
             if ud.lockfile:
                 lf = bb.utils.lockfile(ud.lockfile)
 
@@ -1775,6 +1753,7 @@
 from . import osc
 from . import repo
 from . import clearcase
+from . import npm
 
 methods.append(local.Local())
 methods.append(wget.Wget())
@@ -1791,3 +1770,4 @@
 methods.append(osc.Osc())
 methods.append(repo.Repo())
 methods.append(clearcase.ClearCase())
+methods.append(npm.Npm())
diff --git a/yocto-poky/bitbake/lib/bb/fetch2/git.py b/yocto-poky/bitbake/lib/bb/fetch2/git.py
index 9bd87ad..526668b 100644
--- a/yocto-poky/bitbake/lib/bb/fetch2/git.py
+++ b/yocto-poky/bitbake/lib/bb/fetch2/git.py
@@ -70,6 +70,7 @@
 import os
 import re
 import bb
+import errno
 from   bb    import data
 from   bb.fetch2 import FetchMethod
 from   bb.fetch2 import runfetchcmd
@@ -261,23 +262,7 @@
         if ud.bareclone:
             cloneflags += " --mirror"
 
-        # Versions of git prior to 1.7.9.2 have issues where foo.git and foo get confused
-        # and you end up with some horrible union of the two when you attempt to clone it
-        # The least invasive workaround seems to be a symlink to the real directory to
-        # fool git into ignoring any .git version that may also be present.
-        #
-        # The issue is fixed in more recent versions of git so we can drop this hack in future
-        # when that version becomes common enough.
-        clonedir = ud.clonedir
-        if not ud.path.endswith(".git"):
-            indirectiondir = destdir[:-1] + ".indirectionsymlink"
-            if os.path.exists(indirectiondir):
-                os.remove(indirectiondir)
-            bb.utils.mkdirhier(os.path.dirname(indirectiondir))
-            os.symlink(ud.clonedir, indirectiondir)
-            clonedir = indirectiondir
-
-        runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, cloneflags, clonedir, destdir), d)
+        runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, cloneflags, ud.clonedir, destdir), d)
         os.chdir(destdir)
         repourl = self._get_repo_url(ud)
         runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, repourl), d)
@@ -379,7 +364,7 @@
         """
         pupver = ('', '')
 
-        tagregex = re.compile(d.getVar('GITTAGREGEX', True) or "(?P<pver>([0-9][\.|_]?)+)")
+        tagregex = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX', True) or "(?P<pver>([0-9][\.|_]?)+)")
         try:
             output = self._lsremote(ud, d, "refs/tags/*")
         except bb.fetch2.FetchError or bb.fetch2.NetworkAccess:
diff --git a/yocto-poky/bitbake/lib/bb/fetch2/gitsm.py b/yocto-poky/bitbake/lib/bb/fetch2/gitsm.py
index 0392e48..752f1d3 100644
--- a/yocto-poky/bitbake/lib/bb/fetch2/gitsm.py
+++ b/yocto-poky/bitbake/lib/bb/fetch2/gitsm.py
@@ -110,8 +110,7 @@
         os.chdir(tmpclonedir)
         runfetchcmd(ud.basecmd + " reset --hard", d)
         runfetchcmd(ud.basecmd + " checkout " + ud.revisions[ud.names[0]], d)
-        runfetchcmd(ud.basecmd + " submodule init", d)
-        runfetchcmd(ud.basecmd + " submodule update", d)
+        runfetchcmd(ud.basecmd + " submodule update --init --recursive", d)
         self._set_relative_paths(tmpclonedir)
         runfetchcmd("sed " + gitdir + "/config -i -e 's/bare.*=.*false/bare = true/'", d)
         os.rename(gitdir, ud.clonedir,)
@@ -131,7 +130,5 @@
         os.chdir(ud.destdir)
         submodules = self.uses_submodules(ud, d)
         if submodules:
-            runfetchcmd("cp -r " + ud.clonedir + "/modules " + ud.destdir + "/.git/", d)
-            runfetchcmd(ud.basecmd + " submodule init", d)
-            runfetchcmd(ud.basecmd + " submodule update", d)
-
+            runfetchcmd(ud.basecmd + " checkout " + ud.revisions[ud.names[0]], d)
+            runfetchcmd(ud.basecmd + " submodule update --init --recursive", d)
diff --git a/yocto-poky/bitbake/lib/bb/fetch2/local.py b/yocto-poky/bitbake/lib/bb/fetch2/local.py
index 2d921f7..303a52b 100644
--- a/yocto-poky/bitbake/lib/bb/fetch2/local.py
+++ b/yocto-poky/bitbake/lib/bb/fetch2/local.py
@@ -45,6 +45,7 @@
         ud.decodedurl = urllib.unquote(ud.url.split("://")[1].split(";")[0])
         ud.basename = os.path.basename(ud.decodedurl)
         ud.basepath = ud.decodedurl
+        ud.needdonestamp = False
         return
 
     def localpath(self, urldata, d):
diff --git a/yocto-poky/bitbake/lib/bb/fetch2/npm.py b/yocto-poky/bitbake/lib/bb/fetch2/npm.py
new file mode 100644
index 0000000..e8d9b11
--- /dev/null
+++ b/yocto-poky/bitbake/lib/bb/fetch2/npm.py
@@ -0,0 +1,284 @@
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+"""
+BitBake 'Fetch' NPM implementation
+
+The NPM fetcher is used to retrieve files from the npmjs repository
+
+Usage in the recipe:
+
+    SRC_URI = "npm://registry.npmjs.org/;name=${PN};version=${PV}"
+    Suported SRC_URI options are:
+
+    - name
+    - version
+
+    npm://registry.npmjs.org/${PN}/-/${PN}-${PV}.tgz  would become npm://registry.npmjs.org;name=${PN};ver=${PV}
+    The fetcher all triggers off the existence of ud.localpath. If that exists and has the ".done" stamp, its assumed the fetch is good/done
+
+"""
+
+import os
+import sys
+import urllib
+import json
+import subprocess
+import signal
+import bb
+from   bb import data
+from   bb.fetch2 import FetchMethod
+from   bb.fetch2 import FetchError
+from   bb.fetch2 import ChecksumError
+from   bb.fetch2 import runfetchcmd
+from   bb.fetch2 import logger
+from   bb.fetch2 import UnpackError
+from   bb.fetch2 import ParameterError
+from   distutils import spawn
+
+def subprocess_setup():
+    # Python installs a SIGPIPE handler by default. This is usually not what
+    # non-Python subprocesses expect.
+    # SIGPIPE errors are known issues with gzip/bash
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+
+class Npm(FetchMethod):
+
+    """Class to fetch urls via 'npm'"""
+    def init(self, d):
+        pass
+
+    def supports(self, ud, d):
+        """
+        Check to see if a given url can be fetched with npm
+        """
+        return ud.type in ['npm']
+
+    def debug(self, msg):
+        logger.debug(1, "NpmFetch: %s", msg)
+
+    def clean(self, ud, d):
+        logger.debug(2, "Calling cleanup %s" % ud.pkgname)
+        bb.utils.remove(ud.localpath, False)
+        bb.utils.remove(ud.pkgdatadir, True)
+        bb.utils.remove(ud.fullmirror, False)
+
+    def urldata_init(self, ud, d):
+        """
+        init NPM specific variable within url data
+        """
+        if 'downloadfilename' in ud.parm:
+            ud.basename = ud.parm['downloadfilename']
+        else:
+            ud.basename = os.path.basename(ud.path)
+
+        # can't call it ud.name otherwise fetcher base class will start doing sha1stuff
+        # TODO: find a way to get an sha1/sha256 manifest of pkg & all deps
+        ud.pkgname = ud.parm.get("name", None)
+        if not ud.pkgname:
+            raise ParameterError("NPM fetcher requires a name parameter", ud.url)
+        ud.version = ud.parm.get("version", None)
+        if not ud.version:
+            raise ParameterError("NPM fetcher requires a version parameter", ud.url)
+        ud.bbnpmmanifest = "%s-%s.deps.json" % (ud.pkgname, ud.version)
+        ud.registry = "http://%s" % (ud.url.replace('npm://', '', 1).split(';'))[0]
+        prefixdir = "npm/%s" % ud.pkgname
+        ud.pkgdatadir = d.expand("${DL_DIR}/%s" % prefixdir)
+        if not os.path.exists(ud.pkgdatadir):
+            bb.utils.mkdirhier(ud.pkgdatadir)
+        ud.localpath = d.expand("${DL_DIR}/npm/%s" % ud.bbnpmmanifest)
+
+        self.basecmd = d.getVar("FETCHCMD_wget", True) or "/usr/bin/env wget -O -t 2 -T 30 -nv --passive-ftp --no-check-certificate "
+        self.basecmd += " --directory-prefix=%s " % prefixdir
+
+        ud.write_tarballs = ((data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True) or "0") != "0")
+        ud.mirrortarball = 'npm_%s-%s.tar.xz' % (ud.pkgname, ud.version)
+        ud.fullmirror = os.path.join(d.getVar("DL_DIR", True), ud.mirrortarball)
+
+    def need_update(self, ud, d):
+        if os.path.exists(ud.localpath):
+            return False
+        return True
+
+    def _runwget(self, ud, d, command, quiet):
+        logger.debug(2, "Fetching %s using command '%s'" % (ud.url, command))
+        bb.fetch2.check_network_access(d, command)
+        runfetchcmd(command, d, quiet)
+
+    def _unpackdep(self, ud, pkg, data, destdir, dldir, d):
+        file = data[pkg]['tgz']
+        logger.debug(2, "file to extract is %s" % file)
+        if file.endswith('.tgz') or file.endswith('.tar.gz') or file.endswith('.tar.Z'):
+            cmd = 'tar xz --strip 1 --no-same-owner --warning=no-unknown-keyword -f %s/%s' % (dldir, file)
+        else:
+            bb.fatal("NPM package %s downloaded not a tarball!" % file)
+
+        # Change to subdir before executing command
+        save_cwd = os.getcwd()
+        if not os.path.exists(destdir):
+            os.makedirs(destdir)
+        os.chdir(destdir)
+        path = d.getVar('PATH', True)
+        if path:
+            cmd = "PATH=\"%s\" %s" % (path, cmd)
+        bb.note("Unpacking %s to %s/" % (file, os.getcwd()))
+        ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True)
+        os.chdir(save_cwd)
+
+        if ret != 0:
+            raise UnpackError("Unpack command %s failed with return value %s" % (cmd, ret), ud.url)
+
+        if 'deps' not in data[pkg]:
+            return
+        for dep in data[pkg]['deps']:
+            self._unpackdep(ud, dep, data[pkg]['deps'], "%s/node_modules/%s" % (destdir, dep), dldir, d)
+
+
+    def unpack(self, ud, destdir, d):
+        dldir = d.getVar("DL_DIR", True)
+        depdumpfile = "%s-%s.deps.json" % (ud.pkgname, ud.version)
+        with open("%s/npm/%s" % (dldir, depdumpfile)) as datafile:
+            workobj = json.load(datafile)
+        dldir = "%s/%s" % (os.path.dirname(ud.localpath), ud.pkgname)
+
+        self._unpackdep(ud, ud.pkgname, workobj,  "%s/npmpkg" % destdir, dldir, d)
+
+    def _parse_view(self, output):
+        '''
+        Parse the output of npm view --json; the last JSON result
+        is assumed to be the one that we're interested in.
+        '''
+        pdata = None
+        outdeps = {}
+        datalines = []
+        bracelevel = 0
+        for line in output.splitlines():
+            if bracelevel:
+                datalines.append(line)
+            elif '{' in line:
+                datalines = []
+                datalines.append(line)
+            bracelevel = bracelevel + line.count('{') - line.count('}')
+        if datalines:
+            pdata = json.loads('\n'.join(datalines))
+        return pdata
+
+    def _getdependencies(self, pkg, data, version, d, ud, optional=False):
+        pkgfullname = pkg
+        if version != '*' and not '/' in version:
+            pkgfullname += "@'%s'" % version
+        logger.debug(2, "Calling getdeps on %s" % pkg)
+        fetchcmd = "npm view %s --json --registry %s" % (pkgfullname, ud.registry)
+        output = runfetchcmd(fetchcmd, d, True)
+        pdata = self._parse_view(output)
+        if not pdata:
+            raise FetchError("The command '%s' returned no output" % fetchcmd)
+        if optional:
+            pkg_os = pdata.get('os', None)
+            if pkg_os:
+                if not isinstance(pkg_os, list):
+                    pkg_os = [pkg_os]
+                if 'linux' not in pkg_os or '!linux' in pkg_os:
+                    logger.debug(2, "Skipping %s since it's incompatible with Linux" % pkg)
+                    return
+        #logger.debug(2, "Output URL is %s - %s - %s" % (ud.basepath, ud.basename, ud.localfile))
+        outputurl = pdata['dist']['tarball']
+        data[pkg] = {}
+        data[pkg]['tgz'] = os.path.basename(outputurl)
+        self._runwget(ud, d, "%s %s" % (self.basecmd, outputurl), False)
+
+        dependencies = pdata.get('dependencies', {})
+        optionalDependencies = pdata.get('optionalDependencies', {})
+        depsfound = {}
+        optdepsfound = {}
+        data[pkg]['deps'] = {}
+        for dep in dependencies:
+            if dep in optionalDependencies:
+                optdepsfound[dep] = dependencies[dep]
+            else:
+                depsfound[dep] = dependencies[dep]
+        for dep, version in optdepsfound.iteritems():
+            self._getdependencies(dep, data[pkg]['deps'], version, d, ud, optional=True)
+        for dep, version in depsfound.iteritems():
+            self._getdependencies(dep, data[pkg]['deps'], version, d, ud)
+
+    def _getshrinkeddependencies(self, pkg, data, version, d, ud, lockdown, manifest):
+        logger.debug(2, "NPM shrinkwrap file is %s" % data)
+        outputurl = "invalid"
+        if ('resolved' not in data) or (not data['resolved'].startswith('http')):
+            # will be the case for ${PN}
+            fetchcmd = "npm view %s@%s dist.tarball --registry %s" % (pkg, version, ud.registry)
+            logger.debug(2, "Found this matching URL: %s" % str(fetchcmd))
+            outputurl = runfetchcmd(fetchcmd, d, True)
+        else:
+            outputurl = data['resolved']
+        self._runwget(ud, d, "%s %s" % (self.basecmd, outputurl), False)
+        manifest[pkg] = {}
+        manifest[pkg]['tgz'] = os.path.basename(outputurl).rstrip()
+        manifest[pkg]['deps'] = {}
+
+        if pkg in lockdown:
+            sha1_expected = lockdown[pkg][version]
+            sha1_data = bb.utils.sha1_file("npm/%s/%s" % (ud.pkgname, manifest[pkg]['tgz']))
+            if sha1_expected != sha1_data:
+                msg = "\nFile: '%s' has %s checksum %s when %s was expected" % (manifest[pkg]['tgz'], 'sha1', sha1_data, sha1_expected)
+                raise ChecksumError('Checksum mismatch!%s' % msg)
+        else:
+            logger.debug(2, "No lockdown data for %s@%s" % (pkg, version))
+
+        if 'dependencies' in data:
+            for obj in data['dependencies']:
+                logger.debug(2, "Found dep is %s" % str(obj))
+                self._getshrinkeddependencies(obj, data['dependencies'][obj], data['dependencies'][obj]['version'], d, ud, lockdown, manifest[pkg]['deps'])
+
+    def download(self, ud, d):
+        """Fetch url"""
+        jsondepobj = {}
+        shrinkobj = {}
+        lockdown = {}
+
+        if not os.listdir(ud.pkgdatadir) and os.path.exists(ud.fullmirror):
+            dest = d.getVar("DL_DIR", True)
+            bb.utils.mkdirhier(dest)
+            save_cwd = os.getcwd()
+            os.chdir(dest)
+            runfetchcmd("tar -xJf %s" % (ud.fullmirror), d)
+            os.chdir(save_cwd)
+            return
+
+        shwrf = d.getVar('NPM_SHRINKWRAP', True)
+        logger.debug(2, "NPM shrinkwrap file is %s" % shwrf)
+        try:
+            with open(shwrf) as datafile:
+                shrinkobj = json.load(datafile)
+        except:
+            logger.warn('Missing shrinkwrap file in NPM_SHRINKWRAP for %s, this will lead to unreliable builds!' % ud.pkgname)
+        lckdf = d.getVar('NPM_LOCKDOWN', True)
+        logger.debug(2, "NPM lockdown file is %s" % lckdf)
+        try:
+            with open(lckdf) as datafile:
+                lockdown = json.load(datafile)
+        except:
+            logger.warn('Missing lockdown file in NPM_LOCKDOWN for %s, this will lead to unreproducible builds!' % ud.pkgname)
+
+        if ('name' not in shrinkobj):
+            self._getdependencies(ud.pkgname, jsondepobj, ud.version, d, ud)
+        else:
+            self._getshrinkeddependencies(ud.pkgname, shrinkobj, ud.version, d, ud, lockdown, jsondepobj)
+
+        with open(ud.localpath, 'w') as outfile:
+            json.dump(jsondepobj, outfile)
+
+    def build_mirror_data(self, ud, d):
+        # Generate a mirror tarball if needed
+        if ud.write_tarballs and not os.path.exists(ud.fullmirror):
+            # it's possible that this symlink points to read-only filesystem with PREMIRROR
+            if os.path.islink(ud.fullmirror):
+                os.unlink(ud.fullmirror)
+
+            save_cwd = os.getcwd()
+            os.chdir(d.getVar("DL_DIR", True))
+            logger.info("Creating tarball of npm data")
+            runfetchcmd("tar -cJf %s npm/%s npm/%s" % (ud.fullmirror, ud.bbnpmmanifest, ud.pkgname), d)
+            runfetchcmd("touch %s.done" % (ud.fullmirror), d)
+            os.chdir(save_cwd)
+
diff --git a/yocto-poky/bitbake/lib/bb/fetch2/osc.py b/yocto-poky/bitbake/lib/bb/fetch2/osc.py
index 3d87796..d051dfd 100644
--- a/yocto-poky/bitbake/lib/bb/fetch2/osc.py
+++ b/yocto-poky/bitbake/lib/bb/fetch2/osc.py
@@ -34,13 +34,13 @@
 
         # Create paths to osc checkouts
         relpath = self._strip_leading_slashes(ud.path)
-        ud.pkgdir = os.path.join(data.expand('${OSCDIR}', d), ud.host)
+        ud.pkgdir = os.path.join(d.getVar('OSCDIR', True), ud.host)
         ud.moddir = os.path.join(ud.pkgdir, relpath, ud.module)
 
         if 'rev' in ud.parm:
             ud.revision = ud.parm['rev']
         else:
-            pv = data.getVar("PV", d, 0)
+            pv = d.getVar("PV", False)
             rev = bb.fetch2.srcrev_internal_helper(ud, d)
             if rev and rev != True:
                 ud.revision = rev
@@ -84,7 +84,7 @@
 
         logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'")
 
-        if os.access(os.path.join(data.expand('${OSCDIR}', d), ud.path, ud.module), os.R_OK):
+        if os.access(os.path.join(d.getVar('OSCDIR', True), ud.path, ud.module), os.R_OK):
             oscupdatecmd = self._buildosccommand(ud, d, "update")
             logger.info("Update "+ ud.url)
             # update sources there
@@ -114,7 +114,7 @@
         Generate a .oscrc to be used for this run.
         """
 
-        config_path = os.path.join(data.expand('${OSCDIR}', d), "oscrc")
+        config_path = os.path.join(d.getVar('OSCDIR', True), "oscrc")
         if (os.path.exists(config_path)):
             os.remove(config_path)
 
@@ -123,8 +123,8 @@
         f.write("apisrv = %s\n" % ud.host)
         f.write("scheme = http\n")
         f.write("su-wrapper = su -c\n")
-        f.write("build-root = %s\n" % data.expand('${WORKDIR}', d))
-        f.write("urllist = http://moblin-obs.jf.intel.com:8888/build/%(project)s/%(repository)s/%(buildarch)s/:full/%(name)s.rpm\n")
+        f.write("build-root = %s\n" % d.getVar('WORKDIR', True))
+        f.write("urllist = %s\n" % d.getVar("OSCURLLIST", True))
         f.write("extra-pkgs = gzip\n")
         f.write("\n")
         f.write("[%s]\n" % ud.host)
diff --git a/yocto-poky/bitbake/lib/bb/fetch2/wget.py b/yocto-poky/bitbake/lib/bb/fetch2/wget.py
index bd2a897..8bc9e93 100644
--- a/yocto-poky/bitbake/lib/bb/fetch2/wget.py
+++ b/yocto-poky/bitbake/lib/bb/fetch2/wget.py
@@ -37,7 +37,9 @@
 from   bb.fetch2 import FetchError
 from   bb.fetch2 import logger
 from   bb.fetch2 import runfetchcmd
+from   bb.utils import export_proxies
 from   bs4 import BeautifulSoup
+from   bs4 import SoupStrainer
 
 class Wget(FetchMethod):
     """Class to fetch urls via 'wget'"""
@@ -61,6 +63,8 @@
             ud.basename = os.path.basename(ud.path)
 
         ud.localfile = data.expand(urllib.unquote(ud.basename), d)
+        if not ud.localfile:
+            ud.localfile = data.expand(urllib.unquote(ud.host + ud.path).replace("/", "."), d)
 
         self.basecmd = d.getVar("FETCHCMD_wget", True) or "/usr/bin/env wget -t 2 -T 30 -nv --passive-ftp --no-check-certificate"
 
@@ -218,54 +222,64 @@
 
                 return resp
 
-        def export_proxies(d):
-            variables = ['http_proxy', 'HTTP_PROXY', 'https_proxy', 'HTTPS_PROXY',
-                            'ftp_proxy', 'FTP_PROXY', 'no_proxy', 'NO_PROXY']
-            exported = False
+        class HTTPMethodFallback(urllib2.BaseHandler):
+            """
+            Fallback to GET if HEAD is not allowed (405 HTTP error)
+            """
+            def http_error_405(self, req, fp, code, msg, headers):
+                fp.read()
+                fp.close()
 
-            for v in variables:
-                if v in os.environ.keys():
-                    exported = True
-                else:
-                    v_proxy = d.getVar(v, True)
-                    if v_proxy is not None:
-                        os.environ[v] = v_proxy
-                        exported = True
+                newheaders = dict((k,v) for k,v in req.headers.items()
+                                  if k.lower() not in ("content-length", "content-type"))
+                return self.parent.open(urllib2.Request(req.get_full_url(),
+                                                        headers=newheaders,
+                                                        origin_req_host=req.get_origin_req_host(),
+                                                        unverifiable=True))
 
-            return exported
+            """
+            Some servers (e.g. GitHub archives, hosted on Amazon S3) return 403
+            Forbidden when they actually mean 405 Method Not Allowed.
+            """
+            http_error_403 = http_error_405
 
-        def head_method(self):
-            return "HEAD"
+            """
+            Some servers (e.g. FusionForge) returns 406 Not Acceptable when they
+            actually mean 405 Method Not Allowed.
+            """
+            http_error_406 = http_error_405
 
+        class FixedHTTPRedirectHandler(urllib2.HTTPRedirectHandler):
+            """
+            urllib2.HTTPRedirectHandler resets the method to GET on redirect,
+            when we want to follow redirects using the original method.
+            """
+            def redirect_request(self, req, fp, code, msg, headers, newurl):
+                newreq = urllib2.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, headers, newurl)
+                newreq.get_method = lambda: req.get_method()
+                return newreq
         exported_proxies = export_proxies(d)
 
+        handlers = [FixedHTTPRedirectHandler, HTTPMethodFallback]
+        if export_proxies:
+            handlers.append(urllib2.ProxyHandler())
+        handlers.append(CacheHTTPHandler())
         # XXX: Since Python 2.7.9 ssl cert validation is enabled by default
         # see PEP-0476, this causes verification errors on some https servers
         # so disable by default.
         import ssl
-        ssl_context = None
         if hasattr(ssl, '_create_unverified_context'):
-            ssl_context = ssl._create_unverified_context()
-
-        if exported_proxies == True and ssl_context is not None:
-            opener = urllib2.build_opener(urllib2.ProxyHandler, CacheHTTPHandler,
-                    urllib2.HTTPSHandler(context=ssl_context))
-        elif exported_proxies == False and ssl_context is not None:
-            opener = urllib2.build_opener(CacheHTTPHandler,
-                    urllib2.HTTPSHandler(context=ssl_context))
-        elif exported_proxies == True and ssl_context is None:
-            opener = urllib2.build_opener(urllib2.ProxyHandler, CacheHTTPHandler)
-        else:
-            opener = urllib2.build_opener(CacheHTTPHandler)
-
-        urllib2.Request.get_method = head_method
-        urllib2.install_opener(opener)
-
-        uri = ud.url.split(";")[0]
+            handlers.append(urllib2.HTTPSHandler(context=ssl._create_unverified_context()))
+        opener = urllib2.build_opener(*handlers)
 
         try:
-            urllib2.urlopen(uri)
-        except:
+            uri = ud.url.split(";")[0]
+            r = urllib2.Request(uri)
+            r.get_method = lambda: "HEAD"
+            opener.open(r)
+        except urllib2.URLError as e:
+            # debug for now to avoid spamming the logs in e.g. remote sstate searches
+            logger.debug(2, "checkstatus() urlopen failed: %s" % e)
             return False
         return True
 
@@ -367,7 +381,7 @@
         version = ['', '', '']
 
         bb.debug(3, "VersionURL: %s" % (url))
-        soup = BeautifulSoup(self._fetch_index(url, ud, d))
+        soup = BeautifulSoup(self._fetch_index(url, ud, d), "html.parser", parse_only=SoupStrainer("a"))
         if not soup:
             bb.debug(3, "*** %s NO SOUP" % (url))
             return ""
@@ -406,10 +420,10 @@
         version_dir = ['', '', '']
         version = ['', '', '']
 
-        dirver_regex = re.compile("(\D*)((\d+[\.\-_])+(\d+))")
+        dirver_regex = re.compile("(?P<pfx>\D*)(?P<ver>(\d+[\.\-_])+(\d+))")
         s = dirver_regex.search(dirver)
         if s:
-            version_dir[1] = s.group(2)
+            version_dir[1] = s.group('ver')
         else:
             version_dir[1] = dirver
 
@@ -417,16 +431,26 @@
                 ud.path.split(dirver)[0], ud.user, ud.pswd, {}])
         bb.debug(3, "DirURL: %s, %s" % (dirs_uri, package))
 
-        soup = BeautifulSoup(self._fetch_index(dirs_uri, ud, d))
+        soup = BeautifulSoup(self._fetch_index(dirs_uri, ud, d), "html.parser", parse_only=SoupStrainer("a"))
         if not soup:
             return version[1]
 
         for line in soup.find_all('a', href=True):
             s = dirver_regex.search(line['href'].strip("/"))
             if s:
-                version_dir_new = ['', s.group(2), '']
+                sver = s.group('ver')
+
+                # When prefix is part of the version directory it need to
+                # ensure that only version directory is used so remove previous
+                # directories if exists.
+                #
+                # Example: pfx = '/dir1/dir2/v' and version = '2.5' the expected
+                # result is v2.5.
+                spfx = s.group('pfx').split('/')[-1]
+
+                version_dir_new = ['', sver, '']
                 if self._vercmp(version_dir, version_dir_new) <= 0:
-                    dirver_new = s.group(1) + s.group(2)
+                    dirver_new = spfx + sver
                     path = ud.path.replace(dirver, dirver_new, True) \
                         .split(package)[0]
                     uri = bb.fetch.encodeurl([ud.type, ud.host, path,
@@ -480,7 +504,7 @@
         self.suffix_regex_comp = re.compile(psuffix_regex)
 
         # compile regex, can be specific by package or generic regex
-        pn_regex = d.getVar('REGEX', True)
+        pn_regex = d.getVar('UPSTREAM_CHECK_REGEX', True)
         if pn_regex:
             package_custom_regex_comp = re.compile(pn_regex)
         else:
@@ -516,7 +540,7 @@
         bb.debug(3, "latest_versionstring, regex: %s" % (package_regex.pattern))
 
         uri = ""
-        regex_uri = d.getVar("REGEX_URI", True)
+        regex_uri = d.getVar("UPSTREAM_CHECK_URI", True)
         if not regex_uri:
             path = ud.path.split(package)[0]