Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
| 2 | |
| 3 | # Buildtools and buildtools extended installer helper script |
| 4 | # |
| 5 | # Copyright (C) 2017-2020 Intel Corporation |
| 6 | # |
| 7 | # SPDX-License-Identifier: GPL-2.0-only |
| 8 | # |
| 9 | # NOTE: --with-extended-buildtools is on by default |
| 10 | # |
| 11 | # Example usage (extended buildtools from milestone): |
| 12 | # (1) using --url and --filename |
| 13 | # $ install-buildtools \ |
| 14 | # --url http://downloads.yoctoproject.org/releases/yocto/milestones/yocto-3.1_M3/buildtools \ |
| 15 | # --filename x86_64-buildtools-extended-nativesdk-standalone-3.0+snapshot-20200315.sh |
| 16 | # (2) using --base-url, --release, --installer-version and --build-date |
| 17 | # $ install-buildtools \ |
| 18 | # --base-url http://downloads.yoctoproject.org/releases/yocto \ |
| 19 | # --release yocto-3.1_M3 \ |
| 20 | # --installer-version 3.0+snapshot |
| 21 | # --build-date 202000315 |
| 22 | # |
| 23 | # Example usage (standard buildtools from release): |
| 24 | # (3) using --url and --filename |
| 25 | # $ install-buildtools --without-extended-buildtools \ |
| 26 | # --url http://downloads.yoctoproject.org/releases/yocto/yocto-3.0.2/buildtools \ |
| 27 | # --filename x86_64-buildtools-nativesdk-standalone-3.0.2.sh |
| 28 | # (4) using --base-url, --release and --installer-version |
| 29 | # $ install-buildtools --without-extended-buildtools \ |
| 30 | # --base-url http://downloads.yoctoproject.org/releases/yocto \ |
| 31 | # --release yocto-3.0.2 \ |
| 32 | # --installer-version 3.0.2 |
| 33 | # |
| 34 | |
| 35 | import argparse |
| 36 | import logging |
| 37 | import os |
Andrew Geissler | d25ed32 | 2020-06-27 00:28:28 -0500 | [diff] [blame^] | 38 | import platform |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 39 | import re |
| 40 | import shutil |
| 41 | import shlex |
| 42 | import stat |
| 43 | import subprocess |
| 44 | import sys |
| 45 | import tempfile |
| 46 | from urllib.parse import quote |
| 47 | |
| 48 | scripts_path = os.path.dirname(os.path.realpath(__file__)) |
| 49 | lib_path = scripts_path + '/lib' |
| 50 | sys.path = sys.path + [lib_path] |
| 51 | import scriptutils |
| 52 | import scriptpath |
| 53 | |
| 54 | |
| 55 | PROGNAME = 'install-buildtools' |
| 56 | logger = scriptutils.logger_create(PROGNAME, stream=sys.stdout) |
| 57 | |
| 58 | DEFAULT_INSTALL_DIR = os.path.join(os.path.split(scripts_path)[0],'buildtools') |
| 59 | DEFAULT_BASE_URL = 'http://downloads.yoctoproject.org/releases/yocto' |
Andrew Geissler | d25ed32 | 2020-06-27 00:28:28 -0500 | [diff] [blame^] | 60 | DEFAULT_RELEASE = 'yocto-3.2_M1' |
| 61 | DEFAULT_INSTALLER_VERSION = '3.1+snapshot' |
| 62 | DEFAULT_BUILDDATE = '20200617' |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 63 | |
| 64 | # Python version sanity check |
| 65 | if not (sys.version_info.major == 3 and sys.version_info.minor >= 4): |
| 66 | logger.error("This script requires Python 3.4 or greater") |
| 67 | logger.error("You have Python %s.%s" % |
| 68 | (sys.version_info.major, sys.version_info.minor)) |
| 69 | sys.exit(1) |
| 70 | |
| 71 | # The following three functions are copied directly from |
| 72 | # bitbake/lib/bb/utils.py, in order to allow this script |
| 73 | # to run on versions of python earlier than what bitbake |
| 74 | # supports (e.g. less than Python 3.5 for YP 3.1 release) |
| 75 | |
| 76 | def _hasher(method, filename): |
| 77 | import mmap |
| 78 | |
| 79 | with open(filename, "rb") as f: |
| 80 | try: |
| 81 | with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mm: |
| 82 | for chunk in iter(lambda: mm.read(8192), b''): |
| 83 | method.update(chunk) |
| 84 | except ValueError: |
| 85 | # You can't mmap() an empty file so silence this exception |
| 86 | pass |
| 87 | return method.hexdigest() |
| 88 | |
| 89 | |
| 90 | def md5_file(filename): |
| 91 | """ |
| 92 | Return the hex string representation of the MD5 checksum of filename. |
| 93 | """ |
| 94 | import hashlib |
| 95 | return _hasher(hashlib.md5(), filename) |
| 96 | |
| 97 | def sha256_file(filename): |
| 98 | """ |
| 99 | Return the hex string representation of the 256-bit SHA checksum of |
| 100 | filename. |
| 101 | """ |
| 102 | import hashlib |
| 103 | return _hasher(hashlib.sha256(), filename) |
| 104 | |
| 105 | |
| 106 | def main(): |
| 107 | global DEFAULT_INSTALL_DIR |
| 108 | global DEFAULT_BASE_URL |
| 109 | global DEFAULT_RELEASE |
| 110 | global DEFAULT_INSTALLER_VERSION |
| 111 | global DEFAULT_BUILDDATE |
| 112 | filename = "" |
| 113 | release = "" |
| 114 | buildtools_url = "" |
| 115 | install_dir = "" |
Andrew Geissler | d25ed32 | 2020-06-27 00:28:28 -0500 | [diff] [blame^] | 116 | arch = platform.machine() |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 117 | |
| 118 | parser = argparse.ArgumentParser( |
| 119 | description="Buildtools installation helper", |
| 120 | add_help=False) |
| 121 | parser.add_argument('-u', '--url', |
| 122 | help='URL from where to fetch buildtools SDK installer, not ' |
| 123 | 'including filename (optional)\n' |
| 124 | 'Requires --filename.', |
| 125 | action='store') |
| 126 | parser.add_argument('-f', '--filename', |
| 127 | help='filename for the buildtools SDK installer to be installed ' |
| 128 | '(optional)\nRequires --url', |
| 129 | action='store') |
| 130 | parser.add_argument('-d', '--directory', |
| 131 | default=DEFAULT_INSTALL_DIR, |
| 132 | help='directory where buildtools SDK will be installed (optional)', |
| 133 | action='store') |
| 134 | parser.add_argument('-r', '--release', |
| 135 | default=DEFAULT_RELEASE, |
| 136 | help='Yocto Project release string for SDK which will be ' |
| 137 | 'installed (optional)', |
| 138 | action='store') |
| 139 | parser.add_argument('-V', '--installer-version', |
| 140 | default=DEFAULT_INSTALLER_VERSION, |
| 141 | help='version string for the SDK to be installed (optional)', |
| 142 | action='store') |
| 143 | parser.add_argument('-b', '--base-url', |
| 144 | default=DEFAULT_BASE_URL, |
| 145 | help='base URL from which to fetch SDK (optional)', action='store') |
| 146 | parser.add_argument('-t', '--build-date', |
| 147 | default=DEFAULT_BUILDDATE, |
| 148 | help='Build date of pre-release SDK (optional)', action='store') |
| 149 | group = parser.add_mutually_exclusive_group() |
| 150 | group.add_argument('--with-extended-buildtools', action='store_true', |
| 151 | dest='with_extended_buildtools', |
| 152 | default=True, |
| 153 | help='enable extended buildtools tarball (on by default)') |
| 154 | group.add_argument('--without-extended-buildtools', action='store_false', |
| 155 | dest='with_extended_buildtools', |
| 156 | help='disable extended buildtools (traditional buildtools tarball)') |
Andrew Geissler | d25ed32 | 2020-06-27 00:28:28 -0500 | [diff] [blame^] | 157 | group = parser.add_mutually_exclusive_group() |
| 158 | group.add_argument('-c', '--check', help='enable checksum validation', |
| 159 | default=True, action='store_true') |
| 160 | group.add_argument('-n', '--no-check', help='disable checksum validation', |
| 161 | dest="check", action='store_false') |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 162 | parser.add_argument('-D', '--debug', help='enable debug output', |
| 163 | action='store_true') |
| 164 | parser.add_argument('-q', '--quiet', help='print only errors', |
| 165 | action='store_true') |
| 166 | |
| 167 | parser.add_argument('-h', '--help', action='help', |
| 168 | default=argparse.SUPPRESS, |
| 169 | help='show this help message and exit') |
| 170 | |
| 171 | args = parser.parse_args() |
| 172 | |
| 173 | if args.debug: |
| 174 | logger.setLevel(logging.DEBUG) |
| 175 | elif args.quiet: |
| 176 | logger.setLevel(logging.ERROR) |
| 177 | |
| 178 | if args.url and args.filename: |
| 179 | logger.debug("--url and --filename detected. Ignoring --base-url " |
| 180 | "--release --installer-version arguments.") |
| 181 | filename = args.filename |
| 182 | buildtools_url = "%s/%s" % (args.url, filename) |
| 183 | else: |
| 184 | if args.base_url: |
| 185 | base_url = args.base_url |
| 186 | else: |
| 187 | base_url = DEFAULT_BASE_URL |
| 188 | if args.release: |
| 189 | # check if this is a pre-release "milestone" SDK |
| 190 | m = re.search(r"^(?P<distro>[a-zA-Z\-]+)(?P<version>[0-9.]+)(?P<milestone>_M[1-9])$", |
| 191 | args.release) |
| 192 | logger.debug("milestone regex: %s" % m) |
| 193 | if m and m.group('milestone'): |
| 194 | logger.debug("release[distro]: %s" % m.group('distro')) |
| 195 | logger.debug("release[version]: %s" % m.group('version')) |
| 196 | logger.debug("release[milestone]: %s" % m.group('milestone')) |
| 197 | if not args.build_date: |
| 198 | logger.error("Milestone installers require --build-date") |
| 199 | else: |
| 200 | if args.with_extended_buildtools: |
Andrew Geissler | d25ed32 | 2020-06-27 00:28:28 -0500 | [diff] [blame^] | 201 | filename = "%s-buildtools-extended-nativesdk-standalone-%s-%s.sh" % ( |
| 202 | arch, args.installer_version, args.build_date) |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 203 | else: |
Andrew Geissler | d25ed32 | 2020-06-27 00:28:28 -0500 | [diff] [blame^] | 204 | filename = "%s-buildtools-nativesdk-standalone-%s-%s.sh" % ( |
| 205 | arch, args.installer_version, args.build_date) |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 206 | safe_filename = quote(filename) |
| 207 | buildtools_url = "%s/milestones/%s/buildtools/%s" % (base_url, args.release, safe_filename) |
| 208 | # regular release SDK |
| 209 | else: |
| 210 | if args.with_extended_buildtools: |
Andrew Geissler | d25ed32 | 2020-06-27 00:28:28 -0500 | [diff] [blame^] | 211 | filename = "%s-buildtools-extended-nativesdk-standalone-%s.sh" % (arch, args.installer_version) |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 212 | else: |
Andrew Geissler | d25ed32 | 2020-06-27 00:28:28 -0500 | [diff] [blame^] | 213 | filename = "%s-buildtools-nativesdk-standalone-%s.sh" % (arch, args.installer_version) |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 214 | safe_filename = quote(filename) |
| 215 | buildtools_url = "%s/%s/buildtools/%s" % (base_url, args.release, safe_filename) |
| 216 | |
| 217 | tmpsdk_dir = tempfile.mkdtemp() |
| 218 | try: |
| 219 | # Fetch installer |
| 220 | logger.info("Fetching buildtools installer") |
| 221 | tmpbuildtools = os.path.join(tmpsdk_dir, filename) |
| 222 | ret = subprocess.call("wget -q -O %s %s" % |
| 223 | (tmpbuildtools, buildtools_url), shell=True) |
| 224 | if ret != 0: |
| 225 | logger.error("Could not download file from %s" % buildtools_url) |
| 226 | return ret |
| 227 | |
| 228 | # Verify checksum |
| 229 | if args.check: |
| 230 | logger.info("Fetching buildtools installer checksum") |
| 231 | checksum_type = "" |
Andrew Geissler | d25ed32 | 2020-06-27 00:28:28 -0500 | [diff] [blame^] | 232 | for checksum_type in ["md5sum", "sha256sum"]: |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 233 | check_url = "{}.{}".format(buildtools_url, checksum_type) |
| 234 | checksum_filename = "{}.{}".format(filename, checksum_type) |
| 235 | tmpbuildtools_checksum = os.path.join(tmpsdk_dir, checksum_filename) |
| 236 | ret = subprocess.call("wget -q -O %s %s" % |
| 237 | (tmpbuildtools_checksum, check_url), shell=True) |
| 238 | if ret == 0: |
| 239 | break |
| 240 | else: |
| 241 | if ret != 0: |
| 242 | logger.error("Could not download file from %s" % check_url) |
| 243 | return ret |
Andrew Geissler | d25ed32 | 2020-06-27 00:28:28 -0500 | [diff] [blame^] | 244 | regex = re.compile(r"^(?P<checksum>[0-9a-f]+)\s+(?P<path>.*/)?(?P<filename>.*)$") |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 245 | with open(tmpbuildtools_checksum, 'rb') as f: |
| 246 | original = f.read() |
| 247 | m = re.search(regex, original.decode("utf-8")) |
| 248 | logger.debug("checksum regex match: %s" % m) |
| 249 | logger.debug("checksum: %s" % m.group('checksum')) |
| 250 | logger.debug("path: %s" % m.group('path')) |
| 251 | logger.debug("filename: %s" % m.group('filename')) |
| 252 | if filename != m.group('filename'): |
| 253 | logger.error("Filename does not match name in checksum") |
| 254 | return 1 |
| 255 | checksum = m.group('checksum') |
| 256 | if checksum_type == "md5sum": |
| 257 | checksum_value = md5_file(tmpbuildtools) |
| 258 | else: |
| 259 | checksum_value = sha256_file(tmpbuildtools) |
| 260 | if checksum == checksum_value: |
| 261 | logger.info("Checksum success") |
| 262 | else: |
| 263 | logger.error("Checksum %s expected. Actual checksum is %s." % |
| 264 | (checksum, checksum_value)) |
Andrew Geissler | d25ed32 | 2020-06-27 00:28:28 -0500 | [diff] [blame^] | 265 | return 1 |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 266 | |
| 267 | # Make installer executable |
| 268 | logger.info("Making installer executable") |
| 269 | st = os.stat(tmpbuildtools) |
| 270 | os.chmod(tmpbuildtools, st.st_mode | stat.S_IEXEC) |
| 271 | logger.debug(os.stat(tmpbuildtools)) |
| 272 | if args.directory: |
| 273 | install_dir = args.directory |
| 274 | ret = subprocess.call("%s -d %s -y" % |
| 275 | (tmpbuildtools, install_dir), shell=True) |
| 276 | else: |
| 277 | install_dir = "/opt/poky/%s" % args.installer_version |
| 278 | ret = subprocess.call("%s -y" % tmpbuildtools, shell=True) |
| 279 | if ret != 0: |
| 280 | logger.error("Could not run buildtools installer") |
Andrew Geissler | d25ed32 | 2020-06-27 00:28:28 -0500 | [diff] [blame^] | 281 | return ret |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 282 | |
| 283 | # Setup the environment |
| 284 | logger.info("Setting up the environment") |
| 285 | regex = re.compile(r'^(?P<export>export )?(?P<env_var>[A-Z_]+)=(?P<env_val>.+)$') |
Andrew Geissler | d25ed32 | 2020-06-27 00:28:28 -0500 | [diff] [blame^] | 286 | with open("%s/environment-setup-%s-pokysdk-linux" % |
| 287 | (install_dir, arch), 'rb') as f: |
Andrew Geissler | 82c905d | 2020-04-13 13:39:40 -0500 | [diff] [blame] | 288 | for line in f: |
| 289 | match = regex.search(line.decode('utf-8')) |
| 290 | logger.debug("export regex: %s" % match) |
| 291 | if match: |
| 292 | env_var = match.group('env_var') |
| 293 | logger.debug("env_var: %s" % env_var) |
| 294 | env_val = match.group('env_val') |
| 295 | logger.debug("env_val: %s" % env_val) |
| 296 | os.environ[env_var] = env_val |
| 297 | |
| 298 | # Test installation |
| 299 | logger.info("Testing installation") |
| 300 | tool = "" |
| 301 | m = re.search("extended", tmpbuildtools) |
| 302 | logger.debug("extended regex: %s" % m) |
| 303 | if args.with_extended_buildtools and not m: |
| 304 | logger.info("Ignoring --with-extended-buildtools as filename " |
| 305 | "does not contain 'extended'") |
| 306 | if args.with_extended_buildtools and m: |
| 307 | tool = 'gcc' |
| 308 | else: |
| 309 | tool = 'tar' |
| 310 | logger.debug("install_dir: %s" % install_dir) |
| 311 | cmd = shlex.split("/usr/bin/which %s" % tool) |
| 312 | logger.debug("cmd: %s" % cmd) |
| 313 | logger.debug("tool: %s" % tool) |
| 314 | proc = subprocess.Popen(cmd, stdout=subprocess.PIPE) |
| 315 | output, errors = proc.communicate() |
| 316 | logger.debug("proc.args: %s" % proc.args) |
| 317 | logger.debug("proc.communicate(): output %s" % output) |
| 318 | logger.debug("proc.communicate(): errors %s" % errors) |
| 319 | which_tool = output.decode('utf-8') |
| 320 | logger.debug("which %s: %s" % (tool, which_tool)) |
| 321 | ret = proc.returncode |
| 322 | if not which_tool.startswith(install_dir): |
| 323 | logger.error("Something went wrong: %s not found in %s" % |
| 324 | (tool, install_dir)) |
| 325 | if ret != 0: |
| 326 | logger.error("Something went wrong: installation failed") |
| 327 | else: |
| 328 | logger.info("Installation successful. Remember to source the " |
| 329 | "environment setup script now and in any new session.") |
| 330 | return ret |
| 331 | |
| 332 | finally: |
| 333 | # cleanup tmp directory |
| 334 | shutil.rmtree(tmpsdk_dir) |
| 335 | |
| 336 | |
| 337 | if __name__ == '__main__': |
| 338 | try: |
| 339 | ret = main() |
| 340 | except Exception: |
| 341 | ret = 1 |
| 342 | import traceback |
| 343 | |
| 344 | traceback.print_exc() |
| 345 | sys.exit(ret) |