blob: 3aa42a146637ceb198e35d2fe67576712ef964ba [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# Development tool - sdk-update command plugin
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05002#
3# Copyright (C) 2015-2016 Intel Corporation
4#
Brad Bishopc342db32019-05-15 21:57:59 -04005# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05006#
Patrick Williamsc124f4f2015-09-15 14:41:29 -05007
8import os
9import subprocess
10import logging
11import glob
12import shutil
13import errno
14import sys
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050015import tempfile
16import re
17from devtool import exec_build_env_command, setup_tinfoil, parse_recipe, DevtoolError
Patrick Williamsc124f4f2015-09-15 14:41:29 -050018
19logger = logging.getLogger('devtool')
20
Patrick Williamsc124f4f2015-09-15 14:41:29 -050021def parse_locked_sigs(sigfile_path):
22 """Return <pn:task>:<hash> dictionary"""
23 sig_dict = {}
24 with open(sigfile_path) as f:
25 lines = f.readlines()
26 for line in lines:
27 if ':' in line:
28 taskkey, _, hashval = line.rpartition(':')
29 sig_dict[taskkey.strip()] = hashval.split()[0]
30 return sig_dict
31
32def generate_update_dict(sigfile_new, sigfile_old):
33 """Return a dict containing <pn:task>:<hash> which indicates what need to be updated"""
34 update_dict = {}
35 sigdict_new = parse_locked_sigs(sigfile_new)
36 sigdict_old = parse_locked_sigs(sigfile_old)
37 for k in sigdict_new:
38 if k not in sigdict_old:
39 update_dict[k] = sigdict_new[k]
40 continue
41 if sigdict_new[k] != sigdict_old[k]:
42 update_dict[k] = sigdict_new[k]
43 continue
44 return update_dict
45
Patrick Williamsf1e5d692016-03-30 15:21:19 -050046def get_sstate_objects(update_dict, sstate_dir):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050047 """Return a list containing sstate objects which are to be installed"""
48 sstate_objects = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -050049 for k in update_dict:
50 files = set()
51 hashval = update_dict[k]
52 p = sstate_dir + '/' + hashval[:2] + '/*' + hashval + '*.tgz'
53 files |= set(glob.glob(p))
54 p = sstate_dir + '/*/' + hashval[:2] + '/*' + hashval + '*.tgz'
55 files |= set(glob.glob(p))
56 files = list(files)
57 if len(files) == 1:
58 sstate_objects.extend(files)
59 elif len(files) > 1:
60 logger.error("More than one matching sstate object found for %s" % hashval)
61
62 return sstate_objects
63
64def mkdir(d):
65 try:
66 os.makedirs(d)
67 except OSError as e:
68 if e.errno != errno.EEXIST:
69 raise e
70
71def install_sstate_objects(sstate_objects, src_sdk, dest_sdk):
72 """Install sstate objects into destination SDK"""
73 sstate_dir = os.path.join(dest_sdk, 'sstate-cache')
74 if not os.path.exists(sstate_dir):
75 logger.error("Missing sstate-cache directory in %s, it might not be an extensible SDK." % dest_sdk)
76 raise
77 for sb in sstate_objects:
78 dst = sb.replace(src_sdk, dest_sdk)
79 destdir = os.path.dirname(dst)
80 mkdir(destdir)
81 logger.debug("Copying %s to %s" % (sb, dst))
82 shutil.copy(sb, dst)
83
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050084def check_manifest(fn, basepath):
85 import bb.utils
86 changedfiles = []
87 with open(fn, 'r') as f:
88 for line in f:
89 splitline = line.split()
90 if len(splitline) > 1:
91 chksum = splitline[0]
92 fpath = splitline[1]
93 curr_chksum = bb.utils.sha256_file(os.path.join(basepath, fpath))
94 if chksum != curr_chksum:
95 logger.debug('File %s changed: old csum = %s, new = %s' % (os.path.join(basepath, fpath), curr_chksum, chksum))
96 changedfiles.append(fpath)
97 return changedfiles
98
Patrick Williamsc124f4f2015-09-15 14:41:29 -050099def sdk_update(args, config, basepath, workspace):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600100 """Entry point for devtool sdk-update command"""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500101 updateserver = args.updateserver
102 if not updateserver:
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500103 updateserver = config.get('SDK', 'updateserver', '')
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500104 logger.debug("updateserver: %s" % updateserver)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500105
106 # Make sure we are using sdk-update from within SDK
107 logger.debug("basepath = %s" % basepath)
108 old_locked_sig_file_path = os.path.join(basepath, 'conf/locked-sigs.inc')
109 if not os.path.exists(old_locked_sig_file_path):
110 logger.error("Not using devtool's sdk-update command from within an extensible SDK. Please specify correct basepath via --basepath option")
111 return -1
112 else:
113 logger.debug("Found conf/locked-sigs.inc in %s" % basepath)
114
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600115 if not '://' in updateserver:
116 logger.error("Update server must be a URL")
117 return -1
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500118
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500119 layers_dir = os.path.join(basepath, 'layers')
120 conf_dir = os.path.join(basepath, 'conf')
121
122 # Grab variable values
123 tinfoil = setup_tinfoil(config_only=True, basepath=basepath)
124 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500125 stamps_dir = tinfoil.config_data.getVar('STAMPS_DIR')
126 sstate_mirrors = tinfoil.config_data.getVar('SSTATE_MIRRORS')
127 site_conf_version = tinfoil.config_data.getVar('SITE_CONF_VERSION')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500128 finally:
129 tinfoil.shutdown()
130
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600131 tmpsdk_dir = tempfile.mkdtemp()
132 try:
133 os.makedirs(os.path.join(tmpsdk_dir, 'conf'))
134 new_locked_sig_file_path = os.path.join(tmpsdk_dir, 'conf', 'locked-sigs.inc')
135 # Fetch manifest from server
136 tmpmanifest = os.path.join(tmpsdk_dir, 'conf', 'sdk-conf-manifest')
137 ret = subprocess.call("wget -q -O %s %s/conf/sdk-conf-manifest" % (tmpmanifest, updateserver), shell=True)
Brad Bishopd5ae7d92018-06-14 09:52:03 -0700138 if ret != 0:
139 logger.error("Cannot dowload files from %s" % updateserver)
140 return ret
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600141 changedfiles = check_manifest(tmpmanifest, basepath)
142 if not changedfiles:
143 logger.info("Already up-to-date")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500144 return 0
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600145 # Update metadata
146 logger.debug("Updating metadata via git ...")
147 #Check for the status before doing a fetch and reset
148 if os.path.exists(os.path.join(basepath, 'layers/.git')):
149 out = subprocess.check_output("git status --porcelain", shell=True, cwd=layers_dir)
150 if not out:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500151 ret = subprocess.call("git fetch --all; git reset --hard @{u}", shell=True, cwd=layers_dir)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500152 else:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600153 logger.error("Failed to update metadata as there have been changes made to it. Aborting.");
154 logger.error("Changed files:\n%s" % out);
155 return -1
156 else:
157 ret = -1
158 if ret != 0:
159 ret = subprocess.call("git clone %s/layers/.git" % updateserver, shell=True, cwd=tmpsdk_dir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500160 if ret != 0:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600161 logger.error("Updating metadata via git failed")
162 return ret
163 logger.debug("Updating conf files ...")
164 for changedfile in changedfiles:
165 ret = subprocess.call("wget -q -O %s %s/%s" % (changedfile, updateserver, changedfile), shell=True, cwd=tmpsdk_dir)
166 if ret != 0:
167 logger.error("Updating %s failed" % changedfile)
168 return ret
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500169
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600170 # Check if UNINATIVE_CHECKSUM changed
171 uninative = False
172 if 'conf/local.conf' in changedfiles:
173 def read_uninative_checksums(fn):
174 chksumitems = []
175 with open(fn, 'r') as f:
176 for line in f:
177 if line.startswith('UNINATIVE_CHECKSUM'):
178 splitline = re.split(r'[\[\]"\']', line)
179 if len(splitline) > 3:
180 chksumitems.append((splitline[1], splitline[3]))
181 return chksumitems
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500182
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600183 oldsums = read_uninative_checksums(os.path.join(basepath, 'conf/local.conf'))
184 newsums = read_uninative_checksums(os.path.join(tmpsdk_dir, 'conf/local.conf'))
185 if oldsums != newsums:
186 uninative = True
187 for buildarch, chksum in newsums:
188 uninative_file = os.path.join('downloads', 'uninative', chksum, '%s-nativesdk-libc.tar.bz2' % buildarch)
189 mkdir(os.path.join(tmpsdk_dir, os.path.dirname(uninative_file)))
190 ret = subprocess.call("wget -q -O %s %s/%s" % (uninative_file, updateserver, uninative_file), shell=True, cwd=tmpsdk_dir)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500191
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600192 # Ok, all is well at this point - move everything over
193 tmplayers_dir = os.path.join(tmpsdk_dir, 'layers')
194 if os.path.exists(tmplayers_dir):
195 shutil.rmtree(layers_dir)
196 shutil.move(tmplayers_dir, layers_dir)
197 for changedfile in changedfiles:
198 destfile = os.path.join(basepath, changedfile)
199 os.remove(destfile)
200 shutil.move(os.path.join(tmpsdk_dir, changedfile), destfile)
201 os.remove(os.path.join(conf_dir, 'sdk-conf-manifest'))
202 shutil.move(tmpmanifest, conf_dir)
203 if uninative:
204 shutil.rmtree(os.path.join(basepath, 'downloads', 'uninative'))
205 shutil.move(os.path.join(tmpsdk_dir, 'downloads', 'uninative'), os.path.join(basepath, 'downloads'))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500206
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600207 if not sstate_mirrors:
208 with open(os.path.join(conf_dir, 'site.conf'), 'a') as f:
209 f.write('SCONF_VERSION = "%s"\n' % site_conf_version)
210 f.write('SSTATE_MIRRORS_append = " file://.* %s/sstate-cache/PATH \\n "\n' % updateserver)
211 finally:
212 shutil.rmtree(tmpsdk_dir)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500213
214 if not args.skip_prepare:
215 # Find all potentially updateable tasks
216 sdk_update_targets = []
217 tasks = ['do_populate_sysroot', 'do_packagedata']
218 for root, _, files in os.walk(stamps_dir):
219 for fn in files:
220 if not '.sigdata.' in fn:
221 for task in tasks:
222 if '.%s.' % task in fn or '.%s_setscene.' % task in fn:
223 sdk_update_targets.append('%s:%s' % (os.path.basename(root), task))
224 # Run bitbake command for the whole SDK
225 logger.info("Preparing build system... (This may take some time.)")
226 try:
227 exec_build_env_command(config.init_path, basepath, 'bitbake --setscene-only %s' % ' '.join(sdk_update_targets), stderr=subprocess.STDOUT)
228 output, _ = exec_build_env_command(config.init_path, basepath, 'bitbake -n %s' % ' '.join(sdk_update_targets), stderr=subprocess.STDOUT)
229 runlines = []
230 for line in output.splitlines():
231 if 'Running task ' in line:
232 runlines.append(line)
233 if runlines:
234 logger.error('Unexecuted tasks found in preparation log:\n %s' % '\n '.join(runlines))
235 return -1
236 except bb.process.ExecutionError as e:
237 logger.error('Preparation failed:\n%s' % e.stdout)
238 return -1
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500239 return 0
240
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500241def sdk_install(args, config, basepath, workspace):
242 """Entry point for the devtool sdk-install command"""
243
244 import oe.recipeutils
245 import bb.process
246
247 for recipe in args.recipename:
248 if recipe in workspace:
249 raise DevtoolError('recipe %s is a recipe in your workspace' % recipe)
250
251 tasks = ['do_populate_sysroot', 'do_packagedata']
252 stampprefixes = {}
253 def checkstamp(recipe):
254 stampprefix = stampprefixes[recipe]
255 stamps = glob.glob(stampprefix + '*')
256 for stamp in stamps:
257 if '.sigdata.' not in stamp and stamp.startswith((stampprefix + '.', stampprefix + '_setscene.')):
258 return True
259 else:
260 return False
261
262 install_recipes = []
263 tinfoil = setup_tinfoil(config_only=False, basepath=basepath)
264 try:
265 for recipe in args.recipename:
266 rd = parse_recipe(config, tinfoil, recipe, True)
267 if not rd:
268 return 1
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500269 stampprefixes[recipe] = '%s.%s' % (rd.getVar('STAMP'), tasks[0])
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500270 if checkstamp(recipe):
271 logger.info('%s is already installed' % recipe)
272 else:
273 install_recipes.append(recipe)
274 finally:
275 tinfoil.shutdown()
276
277 if install_recipes:
278 logger.info('Installing %s...' % ', '.join(install_recipes))
279 install_tasks = []
280 for recipe in install_recipes:
281 for task in tasks:
282 if recipe.endswith('-native') and 'package' in task:
283 continue
284 install_tasks.append('%s:%s' % (recipe, task))
285 options = ''
286 if not args.allow_build:
287 options += ' --setscene-only'
288 try:
289 exec_build_env_command(config.init_path, basepath, 'bitbake %s %s' % (options, ' '.join(install_tasks)), watch=True)
290 except bb.process.ExecutionError as e:
291 raise DevtoolError('Failed to install %s:\n%s' % (recipe, str(e)))
292 failed = False
293 for recipe in install_recipes:
294 if checkstamp(recipe):
295 logger.info('Successfully installed %s' % recipe)
296 else:
297 raise DevtoolError('Failed to install %s - unavailable' % recipe)
298 failed = True
299 if failed:
300 return 2
301
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500302 try:
303 exec_build_env_command(config.init_path, basepath, 'bitbake build-sysroots', watch=True)
304 except bb.process.ExecutionError as e:
305 raise DevtoolError('Failed to bitbake build-sysroots:\n%s' % (str(e)))
306
307
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500308def register_commands(subparsers, context):
309 """Register devtool subcommands from the sdk plugin"""
310 if context.fixed_setup:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500311 parser_sdk = subparsers.add_parser('sdk-update',
312 help='Update SDK components',
313 description='Updates installed SDK components from a remote server',
314 group='sdk')
315 updateserver = context.config.get('SDK', 'updateserver', '')
316 if updateserver:
317 parser_sdk.add_argument('updateserver', help='The update server to fetch latest SDK components from (default %s)' % updateserver, nargs='?')
318 else:
319 parser_sdk.add_argument('updateserver', help='The update server to fetch latest SDK components from')
320 parser_sdk.add_argument('--skip-prepare', action="store_true", help='Skip re-preparing the build system after updating (for debugging only)')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500321 parser_sdk.set_defaults(func=sdk_update)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500322
323 parser_sdk_install = subparsers.add_parser('sdk-install',
324 help='Install additional SDK components',
325 description='Installs additional recipe development files into the SDK. (You can use "devtool search" to find available recipes.)',
326 group='sdk')
327 parser_sdk_install.add_argument('recipename', help='Name of the recipe to install the development artifacts for', nargs='+')
328 parser_sdk_install.add_argument('-s', '--allow-build', help='Allow building requested item(s) from source', action='store_true')
329 parser_sdk_install.set_defaults(func=sdk_install)