meta-openembedded and poky: subtree updates
Squash of the following due to dependencies among them
and OpenBMC changes:
meta-openembedded: subtree update:d0748372d2..9201611135
meta-openembedded: subtree update:9201611135..17fd382f34
poky: subtree update:9052e5b32a..2e11d97b6c
poky: subtree update:2e11d97b6c..a8544811d7
The change log was too large for the jenkins plugin
to handle therefore it has been removed. Here is
the first and last commit of each subtree:
meta-openembedded:d0748372d2
cppzmq: bump to version 4.6.0
meta-openembedded:17fd382f34
mpv: Remove X11 dependency
poky:9052e5b32a
package_ipk: Remove pointless comment to trigger rebuild
poky:a8544811d7
pbzip2: Fix license warning
Change-Id: If0fc6c37629642ee207a4ca2f7aa501a2c673cd6
Signed-off-by: Andrew Geissler <geissonator@yahoo.com>
diff --git a/poky/bitbake/lib/bb/fetch2/npmsw.py b/poky/bitbake/lib/bb/fetch2/npmsw.py
new file mode 100644
index 0000000..0c3511d
--- /dev/null
+++ b/poky/bitbake/lib/bb/fetch2/npmsw.py
@@ -0,0 +1,255 @@
+# Copyright (C) 2020 Savoir-Faire Linux
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+"""
+BitBake 'Fetch' npm shrinkwrap implementation
+
+npm fetcher support the SRC_URI with format of:
+SRC_URI = "npmsw://some.registry.url;OptionA=xxx;OptionB=xxx;..."
+
+Supported SRC_URI options are:
+
+- dev
+ Set to 1 to also install devDependencies.
+
+- destsuffix
+ Specifies the directory to use to unpack the dependencies (default: ${S}).
+"""
+
+import json
+import os
+import re
+import bb
+from bb.fetch2 import Fetch
+from bb.fetch2 import FetchMethod
+from bb.fetch2 import ParameterError
+from bb.fetch2 import URI
+from bb.fetch2.npm import npm_integrity
+from bb.fetch2.npm import npm_localfile
+from bb.fetch2.npm import npm_unpack
+from bb.utils import is_semver
+
+def foreach_dependencies(shrinkwrap, callback=None, dev=False):
+ """
+ Run a callback for each dependencies of a shrinkwrap file.
+ The callback is using the format:
+ callback(name, params, deptree)
+ with:
+ name = the package name (string)
+ params = the package parameters (dictionary)
+ deptree = the package dependency tree (array of strings)
+ """
+ def _walk_deps(deps, deptree):
+ for name in deps:
+ subtree = [*deptree, name]
+ _walk_deps(deps[name].get("dependencies", {}), subtree)
+ if callback is not None:
+ if deps[name].get("dev", False) and not dev:
+ continue
+ elif deps[name].get("bundled", False):
+ continue
+ callback(name, deps[name], subtree)
+
+ _walk_deps(shrinkwrap.get("dependencies", {}), [])
+
+class NpmShrinkWrap(FetchMethod):
+ """Class to fetch all package from a shrinkwrap file"""
+
+ def supports(self, ud, d):
+ """Check if a given url can be fetched with npmsw"""
+ return ud.type in ["npmsw"]
+
+ def urldata_init(self, ud, d):
+ """Init npmsw specific variables within url data"""
+
+ # Get the 'shrinkwrap' parameter
+ ud.shrinkwrap_file = re.sub(r"^npmsw://", "", ud.url.split(";")[0])
+
+ # Get the 'dev' parameter
+ ud.dev = bb.utils.to_boolean(ud.parm.get("dev"), False)
+
+ # Resolve the dependencies
+ ud.deps = []
+
+ def _resolve_dependency(name, params, deptree):
+ url = None
+ localpath = None
+ extrapaths = []
+ destsubdirs = [os.path.join("node_modules", dep) for dep in deptree]
+ destsuffix = os.path.join(*destsubdirs)
+
+ integrity = params.get("integrity", None)
+ resolved = params.get("resolved", None)
+ version = params.get("version", None)
+
+ # Handle registry sources
+ if is_semver(version) and resolved and integrity:
+ localfile = npm_localfile(name, version)
+
+ uri = URI(resolved)
+ uri.params["downloadfilename"] = localfile
+
+ checksum_name, checksum_expected = npm_integrity(integrity)
+ uri.params[checksum_name] = checksum_expected
+
+ url = str(uri)
+
+ localpath = os.path.join(d.getVar("DL_DIR"), localfile)
+
+ # Create a resolve file to mimic the npm fetcher and allow
+ # re-usability of the downloaded file.
+ resolvefile = localpath + ".resolved"
+
+ bb.utils.mkdirhier(os.path.dirname(resolvefile))
+ with open(resolvefile, "w") as f:
+ f.write(url)
+
+ extrapaths.append(resolvefile)
+
+ # Handle http tarball sources
+ elif version.startswith("http") and integrity:
+ localfile = os.path.join("npm2", os.path.basename(version))
+
+ uri = URI(version)
+ uri.params["downloadfilename"] = localfile
+
+ checksum_name, checksum_expected = npm_integrity(integrity)
+ uri.params[checksum_name] = checksum_expected
+
+ url = str(uri)
+
+ localpath = os.path.join(d.getVar("DL_DIR"), localfile)
+
+ # Handle git sources
+ elif version.startswith("git"):
+ regex = re.compile(r"""
+ ^
+ git\+
+ (?P<protocol>[a-z]+)
+ ://
+ (?P<url>[^#]+)
+ \#
+ (?P<rev>[0-9a-f]+)
+ $
+ """, re.VERBOSE)
+
+ match = regex.match(version)
+
+ if not match:
+ raise ParameterError("Invalid git url: %s" % version, ud.url)
+
+ groups = match.groupdict()
+
+ uri = URI("git://" + str(groups["url"]))
+ uri.params["protocol"] = str(groups["protocol"])
+ uri.params["rev"] = str(groups["rev"])
+ uri.params["destsuffix"] = destsuffix
+
+ url = str(uri)
+
+ # local tarball sources and local link sources are unsupported
+ else:
+ raise ParameterError("Unsupported dependency: %s" % name, ud.url)
+
+ ud.deps.append({
+ "url": url,
+ "localpath": localpath,
+ "extrapaths": extrapaths,
+ "destsuffix": destsuffix,
+ })
+
+ try:
+ with open(ud.shrinkwrap_file, "r") as f:
+ shrinkwrap = json.load(f)
+ except Exception as e:
+ raise ParameterError("Invalid shrinkwrap file: %s" % str(e), ud.url)
+
+ foreach_dependencies(shrinkwrap, _resolve_dependency, ud.dev)
+
+ # Avoid conflicts between the environment data and:
+ # - the proxy url revision
+ # - the proxy url checksum
+ data = bb.data.createCopy(d)
+ data.delVar("SRCREV")
+ data.delVarFlags("SRC_URI")
+
+ # This fetcher resolves multiple URIs from a shrinkwrap file and then
+ # forwards it to a proxy fetcher. The management of the donestamp file,
+ # the lockfile and the checksums are forwarded to the proxy fetcher.
+ ud.proxy = Fetch([dep["url"] for dep in ud.deps], data)
+ ud.needdonestamp = False
+
+ @staticmethod
+ def _foreach_proxy_method(ud, handle):
+ returns = []
+ for proxy_url in ud.proxy.urls:
+ proxy_ud = ud.proxy.ud[proxy_url]
+ proxy_d = ud.proxy.d
+ proxy_ud.setup_localpath(proxy_d)
+ returns.append(handle(proxy_ud.method, proxy_ud, proxy_d))
+ return returns
+
+ def verify_donestamp(self, ud, d):
+ """Verify the donestamp file"""
+ def _handle(m, ud, d):
+ return m.verify_donestamp(ud, d)
+ return all(self._foreach_proxy_method(ud, _handle))
+
+ def update_donestamp(self, ud, d):
+ """Update the donestamp file"""
+ def _handle(m, ud, d):
+ m.update_donestamp(ud, d)
+ self._foreach_proxy_method(ud, _handle)
+
+ def need_update(self, ud, d):
+ """Force a fetch, even if localpath exists ?"""
+ def _handle(m, ud, d):
+ return m.need_update(ud, d)
+ return all(self._foreach_proxy_method(ud, _handle))
+
+ def try_mirrors(self, fetch, ud, d, mirrors):
+ """Try to use a mirror"""
+ def _handle(m, ud, d):
+ return m.try_mirrors(fetch, ud, d, mirrors)
+ return all(self._foreach_proxy_method(ud, _handle))
+
+ def download(self, ud, d):
+ """Fetch url"""
+ ud.proxy.download()
+
+ def unpack(self, ud, rootdir, d):
+ """Unpack the downloaded dependencies"""
+ destdir = d.getVar("S")
+ destsuffix = ud.parm.get("destsuffix")
+ if destsuffix:
+ destdir = os.path.join(rootdir, destsuffix)
+
+ bb.utils.mkdirhier(destdir)
+ bb.utils.copyfile(ud.shrinkwrap_file,
+ os.path.join(destdir, "npm-shrinkwrap.json"))
+
+ auto = [dep["url"] for dep in ud.deps if not dep["localpath"]]
+ manual = [dep for dep in ud.deps if dep["localpath"]]
+
+ if auto:
+ ud.proxy.unpack(destdir, auto)
+
+ for dep in manual:
+ depdestdir = os.path.join(destdir, dep["destsuffix"])
+ npm_unpack(dep["localpath"], depdestdir, d)
+
+ def clean(self, ud, d):
+ """Clean any existing full or partial download"""
+ ud.proxy.clean()
+
+ # Clean extra files
+ for dep in ud.deps:
+ for path in dep["extrapaths"]:
+ bb.utils.remove(path)
+
+ def done(self, ud, d):
+ """Is the download done ?"""
+ def _handle(m, ud, d):
+ return m.done(ud, d)
+ return all(self._foreach_proxy_method(ud, _handle))