meta-openembedded and poky: subtree updates

Squash of the following due to dependencies among them
and OpenBMC changes:

meta-openembedded: subtree update:d0748372d2..9201611135
meta-openembedded: subtree update:9201611135..17fd382f34
poky: subtree update:9052e5b32a..2e11d97b6c
poky: subtree update:2e11d97b6c..a8544811d7

The change log was too large for the jenkins plugin
to handle therefore it has been removed. Here is
the first and last commit of each subtree:

meta-openembedded:d0748372d2
      cppzmq: bump to version 4.6.0
meta-openembedded:17fd382f34
      mpv: Remove X11 dependency
poky:9052e5b32a
      package_ipk: Remove pointless comment to trigger rebuild
poky:a8544811d7
      pbzip2: Fix license warning

Change-Id: If0fc6c37629642ee207a4ca2f7aa501a2c673cd6
Signed-off-by: Andrew Geissler <geissonator@yahoo.com>
diff --git a/poky/scripts/install-buildtools b/poky/scripts/install-buildtools
new file mode 100755
index 0000000..c6b3a1e
--- /dev/null
+++ b/poky/scripts/install-buildtools
@@ -0,0 +1,339 @@
+#!/usr/bin/env python3
+
+# Buildtools and buildtools extended installer helper script
+#
+# Copyright (C) 2017-2020 Intel Corporation
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+#  NOTE: --with-extended-buildtools is on by default
+#
+#  Example usage (extended buildtools from milestone):
+#    (1) using --url and --filename
+#        $ install-buildtools \
+#          --url http://downloads.yoctoproject.org/releases/yocto/milestones/yocto-3.1_M3/buildtools \
+#          --filename x86_64-buildtools-extended-nativesdk-standalone-3.0+snapshot-20200315.sh
+#    (2) using --base-url, --release, --installer-version and --build-date
+#        $ install-buildtools \
+#          --base-url http://downloads.yoctoproject.org/releases/yocto \
+#          --release yocto-3.1_M3 \
+#          --installer-version 3.0+snapshot
+#          --build-date 202000315
+#
+#  Example usage (standard buildtools from release):
+#    (3) using --url and --filename
+#        $ install-buildtools --without-extended-buildtools \
+#          --url http://downloads.yoctoproject.org/releases/yocto/yocto-3.0.2/buildtools \
+#          --filename x86_64-buildtools-nativesdk-standalone-3.0.2.sh
+#    (4) using --base-url, --release and --installer-version
+#        $ install-buildtools --without-extended-buildtools \
+#          --base-url http://downloads.yoctoproject.org/releases/yocto \
+#          --release yocto-3.0.2 \
+#          --installer-version 3.0.2
+#
+
+import argparse
+import logging
+import os
+import re
+import shutil
+import shlex
+import stat
+import subprocess
+import sys
+import tempfile
+from urllib.parse import quote
+
+scripts_path = os.path.dirname(os.path.realpath(__file__))
+lib_path = scripts_path + '/lib'
+sys.path = sys.path + [lib_path]
+import scriptutils
+import scriptpath
+
+
+PROGNAME = 'install-buildtools'
+logger = scriptutils.logger_create(PROGNAME, stream=sys.stdout)
+
+DEFAULT_INSTALL_DIR = os.path.join(os.path.split(scripts_path)[0],'buildtools')
+DEFAULT_BASE_URL = 'http://downloads.yoctoproject.org/releases/yocto'
+DEFAULT_RELEASE = 'yocto-3.1'
+DEFAULT_INSTALLER_VERSION = '3.1'
+DEFAULT_BUILDDATE = ''
+
+# Python version sanity check
+if not (sys.version_info.major == 3 and sys.version_info.minor >= 4):
+    logger.error("This script requires Python 3.4 or greater")
+    logger.error("You have Python %s.%s" %
+	  (sys.version_info.major, sys.version_info.minor))
+    sys.exit(1)
+
+# The following three functions are copied directly from
+# bitbake/lib/bb/utils.py, in order to allow this script
+# to run on versions of python earlier than what bitbake
+# supports (e.g. less than Python 3.5 for YP 3.1 release)
+
+def _hasher(method, filename):
+    import mmap
+
+    with open(filename, "rb") as f:
+        try:
+            with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mm:
+                for chunk in iter(lambda: mm.read(8192), b''):
+                    method.update(chunk)
+        except ValueError:
+            # You can't mmap() an empty file so silence this exception
+            pass
+    return method.hexdigest()
+
+
+def md5_file(filename):
+    """
+    Return the hex string representation of the MD5 checksum of filename.
+    """
+    import hashlib
+    return _hasher(hashlib.md5(), filename)
+
+def sha256_file(filename):
+    """
+    Return the hex string representation of the 256-bit SHA checksum of
+    filename.
+    """
+    import hashlib
+    return _hasher(hashlib.sha256(), filename)
+
+
+def main():
+    global DEFAULT_INSTALL_DIR
+    global DEFAULT_BASE_URL
+    global DEFAULT_RELEASE
+    global DEFAULT_INSTALLER_VERSION
+    global DEFAULT_BUILDDATE
+    filename = ""
+    release = ""
+    buildtools_url = ""
+    install_dir = ""
+
+    parser = argparse.ArgumentParser(
+        description="Buildtools installation helper",
+        add_help=False)
+    parser.add_argument('-u', '--url',
+                        help='URL from where to fetch buildtools SDK installer, not '
+                             'including filename (optional)\n'
+                             'Requires --filename.',
+                        action='store')
+    parser.add_argument('-f', '--filename',
+                        help='filename for the buildtools SDK installer to be installed '
+                             '(optional)\nRequires --url',
+                        action='store')
+    parser.add_argument('-d', '--directory',
+                        default=DEFAULT_INSTALL_DIR,
+                        help='directory where buildtools SDK will be installed (optional)',
+                        action='store')
+    parser.add_argument('-r', '--release',
+                        default=DEFAULT_RELEASE,
+                        help='Yocto Project release string for SDK which will be '
+                             'installed (optional)',
+                        action='store')
+    parser.add_argument('-V', '--installer-version',
+                        default=DEFAULT_INSTALLER_VERSION,
+                        help='version string for the SDK to be installed (optional)',
+                        action='store')
+    parser.add_argument('-b', '--base-url',
+                        default=DEFAULT_BASE_URL,
+                        help='base URL from which to fetch SDK (optional)', action='store')
+    parser.add_argument('-t', '--build-date',
+                        default=DEFAULT_BUILDDATE,
+                        help='Build date of pre-release SDK (optional)', action='store')
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument('--with-extended-buildtools', action='store_true',
+                       dest='with_extended_buildtools',
+                       default=True,
+                       help='enable extended buildtools tarball (on by default)')
+    group.add_argument('--without-extended-buildtools', action='store_false',
+                       dest='with_extended_buildtools',
+                       help='disable extended buildtools (traditional buildtools tarball)')
+    parser.add_argument('-c', '--check', help='enable md5 checksum checking',
+                        default=True,
+                        action='store_true')
+    parser.add_argument('-D', '--debug', help='enable debug output',
+                        action='store_true')
+    parser.add_argument('-q', '--quiet', help='print only errors',
+                        action='store_true')
+
+    parser.add_argument('-h', '--help', action='help',
+                        default=argparse.SUPPRESS,
+                        help='show this help message and exit')
+
+    args = parser.parse_args()
+
+    if args.debug:
+        logger.setLevel(logging.DEBUG)
+    elif args.quiet:
+        logger.setLevel(logging.ERROR)
+
+    if args.url and args.filename:
+        logger.debug("--url and --filename detected. Ignoring --base-url "
+                     "--release --installer-version  arguments.")
+        filename = args.filename
+        buildtools_url = "%s/%s" % (args.url, filename)
+    else:
+        if args.base_url:
+            base_url = args.base_url
+        else:
+            base_url = DEFAULT_BASE_URL
+        if args.release:
+            # check if this is a pre-release "milestone" SDK
+            m = re.search(r"^(?P<distro>[a-zA-Z\-]+)(?P<version>[0-9.]+)(?P<milestone>_M[1-9])$",
+                          args.release)
+            logger.debug("milestone regex: %s" % m)
+            if m and m.group('milestone'):
+                logger.debug("release[distro]: %s" % m.group('distro'))
+                logger.debug("release[version]: %s" % m.group('version'))
+                logger.debug("release[milestone]: %s" % m.group('milestone'))
+                if not args.build_date:
+                    logger.error("Milestone installers require --build-date")
+                else:
+                    if args.with_extended_buildtools:
+                        filename = "x86_64-buildtools-extended-nativesdk-standalone-%s-%s.sh" % (
+                            args.installer_version, args.build_date)
+                    else:
+                        filename = "x86_64-buildtools-nativesdk-standalone-%s-%s.sh" % (
+                            args.installer_version, args.build_date)
+                    safe_filename = quote(filename)
+                    buildtools_url = "%s/milestones/%s/buildtools/%s" % (base_url, args.release, safe_filename)
+            # regular release SDK
+            else:
+                if args.with_extended_buildtools:
+                    filename = "x86_64-buildtools-extended-nativesdk-standalone-%s.sh" % args.installer_version
+                else:
+                    filename = "x86_64-buildtools-nativesdk-standalone-%s.sh" % args.installer_version
+                safe_filename = quote(filename)
+                buildtools_url = "%s/%s/buildtools/%s" % (base_url, args.release, safe_filename)
+
+    tmpsdk_dir = tempfile.mkdtemp()
+    try:
+        # Fetch installer
+        logger.info("Fetching buildtools installer")
+        tmpbuildtools = os.path.join(tmpsdk_dir, filename)
+        ret = subprocess.call("wget -q -O %s %s" %
+                              (tmpbuildtools, buildtools_url), shell=True)
+        if ret != 0:
+            logger.error("Could not download file from %s" % buildtools_url)
+            return ret
+
+        # Verify checksum
+        if args.check:
+            logger.info("Fetching buildtools installer checksum")
+            checksum_type = ""
+            for checksum_type in ["md5sum", "sha256"]:
+                check_url = "{}.{}".format(buildtools_url, checksum_type)
+                checksum_filename = "{}.{}".format(filename, checksum_type)
+                tmpbuildtools_checksum = os.path.join(tmpsdk_dir, checksum_filename)
+                ret = subprocess.call("wget -q -O %s %s" %
+                                      (tmpbuildtools_checksum, check_url), shell=True)
+                if ret == 0:
+                    break
+            else:
+                if ret != 0:
+                    logger.error("Could not download file from %s" % check_url)
+                    return ret
+            regex = re.compile(r"^(?P<checksum>[0-9a-f]+)\s\s(?P<path>.*/)?(?P<filename>.*)$")
+            with open(tmpbuildtools_checksum, 'rb') as f:
+                original = f.read()
+                m = re.search(regex, original.decode("utf-8"))
+                logger.debug("checksum regex match: %s" % m)
+                logger.debug("checksum: %s" % m.group('checksum'))
+                logger.debug("path: %s" % m.group('path'))
+                logger.debug("filename: %s" % m.group('filename'))
+                if filename != m.group('filename'):
+                    logger.error("Filename does not match name in checksum")
+                    return 1
+                checksum = m.group('checksum')
+            if checksum_type == "md5sum":
+                checksum_value = md5_file(tmpbuildtools)
+            else:
+                checksum_value = sha256_file(tmpbuildtools)
+            if checksum == checksum_value:
+                    logger.info("Checksum success")
+            else:
+                logger.error("Checksum %s expected. Actual checksum is %s." %
+                             (checksum, checksum_value))
+
+        # Make installer executable
+        logger.info("Making installer executable")
+        st = os.stat(tmpbuildtools)
+        os.chmod(tmpbuildtools, st.st_mode | stat.S_IEXEC)
+        logger.debug(os.stat(tmpbuildtools))
+        if args.directory:
+            install_dir = args.directory
+            ret = subprocess.call("%s -d %s -y" %
+                                  (tmpbuildtools, install_dir), shell=True)
+        else:
+            install_dir = "/opt/poky/%s" % args.installer_version
+            ret = subprocess.call("%s -y" % tmpbuildtools, shell=True)
+        if ret != 0:
+            logger.error("Could not run buildtools installer")
+
+        # Setup the environment
+        logger.info("Setting up the environment")
+        regex = re.compile(r'^(?P<export>export )?(?P<env_var>[A-Z_]+)=(?P<env_val>.+)$')
+        with open("%s/environment-setup-x86_64-pokysdk-linux" %
+                  install_dir, 'rb') as f:
+            for line in f:
+                match = regex.search(line.decode('utf-8'))
+                logger.debug("export regex: %s" % match)
+                if match:
+                    env_var = match.group('env_var')
+                    logger.debug("env_var: %s" % env_var)
+                    env_val = match.group('env_val')
+                    logger.debug("env_val: %s" % env_val)
+                    os.environ[env_var] = env_val
+
+        # Test installation
+        logger.info("Testing installation")
+        tool = ""
+        m = re.search("extended", tmpbuildtools)
+        logger.debug("extended regex: %s" % m)
+        if args.with_extended_buildtools and not m:
+            logger.info("Ignoring --with-extended-buildtools as filename "
+                        "does not contain 'extended'")
+        if args.with_extended_buildtools and m:
+            tool = 'gcc'
+        else:
+            tool = 'tar'
+        logger.debug("install_dir: %s" % install_dir)
+        cmd = shlex.split("/usr/bin/which %s" % tool)
+        logger.debug("cmd: %s" % cmd)
+        logger.debug("tool: %s" % tool)
+        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+        output, errors = proc.communicate()
+        logger.debug("proc.args: %s" % proc.args)
+        logger.debug("proc.communicate(): output %s" % output)
+        logger.debug("proc.communicate(): errors %s" % errors)
+        which_tool = output.decode('utf-8')
+        logger.debug("which %s: %s" % (tool, which_tool))
+        ret = proc.returncode
+        if not which_tool.startswith(install_dir):
+            logger.error("Something went wrong: %s not found in %s" %
+                         (tool, install_dir))
+        if ret != 0:
+            logger.error("Something went wrong: installation failed")
+        else:
+            logger.info("Installation successful. Remember to source the "
+                        "environment setup script now and in any new session.")
+        return ret
+
+    finally:
+        # cleanup tmp directory
+        shutil.rmtree(tmpsdk_dir)
+
+
+if __name__ == '__main__':
+    try:
+        ret = main()
+    except Exception:
+        ret = 1
+        import traceback
+
+        traceback.print_exc()
+    sys.exit(ret)