blob: 922277b79f5b183f4914fca37503bf26569c10c0 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# Development tool - sdk-update command plugin
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05002#
3# Copyright (C) 2015-2016 Intel Corporation
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License version 2 as
7# published by the Free Software Foundation.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License along
15# with this program; if not, write to the Free Software Foundation, Inc.,
16# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
Patrick Williamsc124f4f2015-09-15 14:41:29 -050017
18import os
19import subprocess
20import logging
21import glob
22import shutil
23import errno
24import sys
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050025import tempfile
26import re
27from devtool import exec_build_env_command, setup_tinfoil, parse_recipe, DevtoolError
Patrick Williamsc124f4f2015-09-15 14:41:29 -050028
29logger = logging.getLogger('devtool')
30
Patrick Williamsc124f4f2015-09-15 14:41:29 -050031def parse_locked_sigs(sigfile_path):
32 """Return <pn:task>:<hash> dictionary"""
33 sig_dict = {}
34 with open(sigfile_path) as f:
35 lines = f.readlines()
36 for line in lines:
37 if ':' in line:
38 taskkey, _, hashval = line.rpartition(':')
39 sig_dict[taskkey.strip()] = hashval.split()[0]
40 return sig_dict
41
42def generate_update_dict(sigfile_new, sigfile_old):
43 """Return a dict containing <pn:task>:<hash> which indicates what need to be updated"""
44 update_dict = {}
45 sigdict_new = parse_locked_sigs(sigfile_new)
46 sigdict_old = parse_locked_sigs(sigfile_old)
47 for k in sigdict_new:
48 if k not in sigdict_old:
49 update_dict[k] = sigdict_new[k]
50 continue
51 if sigdict_new[k] != sigdict_old[k]:
52 update_dict[k] = sigdict_new[k]
53 continue
54 return update_dict
55
Patrick Williamsf1e5d692016-03-30 15:21:19 -050056def get_sstate_objects(update_dict, sstate_dir):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050057 """Return a list containing sstate objects which are to be installed"""
58 sstate_objects = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -050059 for k in update_dict:
60 files = set()
61 hashval = update_dict[k]
62 p = sstate_dir + '/' + hashval[:2] + '/*' + hashval + '*.tgz'
63 files |= set(glob.glob(p))
64 p = sstate_dir + '/*/' + hashval[:2] + '/*' + hashval + '*.tgz'
65 files |= set(glob.glob(p))
66 files = list(files)
67 if len(files) == 1:
68 sstate_objects.extend(files)
69 elif len(files) > 1:
70 logger.error("More than one matching sstate object found for %s" % hashval)
71
72 return sstate_objects
73
74def mkdir(d):
75 try:
76 os.makedirs(d)
77 except OSError as e:
78 if e.errno != errno.EEXIST:
79 raise e
80
81def install_sstate_objects(sstate_objects, src_sdk, dest_sdk):
82 """Install sstate objects into destination SDK"""
83 sstate_dir = os.path.join(dest_sdk, 'sstate-cache')
84 if not os.path.exists(sstate_dir):
85 logger.error("Missing sstate-cache directory in %s, it might not be an extensible SDK." % dest_sdk)
86 raise
87 for sb in sstate_objects:
88 dst = sb.replace(src_sdk, dest_sdk)
89 destdir = os.path.dirname(dst)
90 mkdir(destdir)
91 logger.debug("Copying %s to %s" % (sb, dst))
92 shutil.copy(sb, dst)
93
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050094def check_manifest(fn, basepath):
95 import bb.utils
96 changedfiles = []
97 with open(fn, 'r') as f:
98 for line in f:
99 splitline = line.split()
100 if len(splitline) > 1:
101 chksum = splitline[0]
102 fpath = splitline[1]
103 curr_chksum = bb.utils.sha256_file(os.path.join(basepath, fpath))
104 if chksum != curr_chksum:
105 logger.debug('File %s changed: old csum = %s, new = %s' % (os.path.join(basepath, fpath), curr_chksum, chksum))
106 changedfiles.append(fpath)
107 return changedfiles
108
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500109def sdk_update(args, config, basepath, workspace):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600110 """Entry point for devtool sdk-update command"""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500111 updateserver = args.updateserver
112 if not updateserver:
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500113 updateserver = config.get('SDK', 'updateserver', '')
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500114 logger.debug("updateserver: %s" % updateserver)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500115
116 # Make sure we are using sdk-update from within SDK
117 logger.debug("basepath = %s" % basepath)
118 old_locked_sig_file_path = os.path.join(basepath, 'conf/locked-sigs.inc')
119 if not os.path.exists(old_locked_sig_file_path):
120 logger.error("Not using devtool's sdk-update command from within an extensible SDK. Please specify correct basepath via --basepath option")
121 return -1
122 else:
123 logger.debug("Found conf/locked-sigs.inc in %s" % basepath)
124
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600125 if not '://' in updateserver:
126 logger.error("Update server must be a URL")
127 return -1
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500128
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500129 layers_dir = os.path.join(basepath, 'layers')
130 conf_dir = os.path.join(basepath, 'conf')
131
132 # Grab variable values
133 tinfoil = setup_tinfoil(config_only=True, basepath=basepath)
134 try:
135 stamps_dir = tinfoil.config_data.getVar('STAMPS_DIR', True)
136 sstate_mirrors = tinfoil.config_data.getVar('SSTATE_MIRRORS', True)
137 site_conf_version = tinfoil.config_data.getVar('SITE_CONF_VERSION', True)
138 finally:
139 tinfoil.shutdown()
140
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600141 tmpsdk_dir = tempfile.mkdtemp()
142 try:
143 os.makedirs(os.path.join(tmpsdk_dir, 'conf'))
144 new_locked_sig_file_path = os.path.join(tmpsdk_dir, 'conf', 'locked-sigs.inc')
145 # Fetch manifest from server
146 tmpmanifest = os.path.join(tmpsdk_dir, 'conf', 'sdk-conf-manifest')
147 ret = subprocess.call("wget -q -O %s %s/conf/sdk-conf-manifest" % (tmpmanifest, updateserver), shell=True)
148 changedfiles = check_manifest(tmpmanifest, basepath)
149 if not changedfiles:
150 logger.info("Already up-to-date")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500151 return 0
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600152 # Update metadata
153 logger.debug("Updating metadata via git ...")
154 #Check for the status before doing a fetch and reset
155 if os.path.exists(os.path.join(basepath, 'layers/.git')):
156 out = subprocess.check_output("git status --porcelain", shell=True, cwd=layers_dir)
157 if not out:
158 ret = subprocess.call("git fetch --all; git reset --hard", shell=True, cwd=layers_dir)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500159 else:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600160 logger.error("Failed to update metadata as there have been changes made to it. Aborting.");
161 logger.error("Changed files:\n%s" % out);
162 return -1
163 else:
164 ret = -1
165 if ret != 0:
166 ret = subprocess.call("git clone %s/layers/.git" % updateserver, shell=True, cwd=tmpsdk_dir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500167 if ret != 0:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600168 logger.error("Updating metadata via git failed")
169 return ret
170 logger.debug("Updating conf files ...")
171 for changedfile in changedfiles:
172 ret = subprocess.call("wget -q -O %s %s/%s" % (changedfile, updateserver, changedfile), shell=True, cwd=tmpsdk_dir)
173 if ret != 0:
174 logger.error("Updating %s failed" % changedfile)
175 return ret
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500176
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600177 # Check if UNINATIVE_CHECKSUM changed
178 uninative = False
179 if 'conf/local.conf' in changedfiles:
180 def read_uninative_checksums(fn):
181 chksumitems = []
182 with open(fn, 'r') as f:
183 for line in f:
184 if line.startswith('UNINATIVE_CHECKSUM'):
185 splitline = re.split(r'[\[\]"\']', line)
186 if len(splitline) > 3:
187 chksumitems.append((splitline[1], splitline[3]))
188 return chksumitems
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500189
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600190 oldsums = read_uninative_checksums(os.path.join(basepath, 'conf/local.conf'))
191 newsums = read_uninative_checksums(os.path.join(tmpsdk_dir, 'conf/local.conf'))
192 if oldsums != newsums:
193 uninative = True
194 for buildarch, chksum in newsums:
195 uninative_file = os.path.join('downloads', 'uninative', chksum, '%s-nativesdk-libc.tar.bz2' % buildarch)
196 mkdir(os.path.join(tmpsdk_dir, os.path.dirname(uninative_file)))
197 ret = subprocess.call("wget -q -O %s %s/%s" % (uninative_file, updateserver, uninative_file), shell=True, cwd=tmpsdk_dir)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500198
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600199 # Ok, all is well at this point - move everything over
200 tmplayers_dir = os.path.join(tmpsdk_dir, 'layers')
201 if os.path.exists(tmplayers_dir):
202 shutil.rmtree(layers_dir)
203 shutil.move(tmplayers_dir, layers_dir)
204 for changedfile in changedfiles:
205 destfile = os.path.join(basepath, changedfile)
206 os.remove(destfile)
207 shutil.move(os.path.join(tmpsdk_dir, changedfile), destfile)
208 os.remove(os.path.join(conf_dir, 'sdk-conf-manifest'))
209 shutil.move(tmpmanifest, conf_dir)
210 if uninative:
211 shutil.rmtree(os.path.join(basepath, 'downloads', 'uninative'))
212 shutil.move(os.path.join(tmpsdk_dir, 'downloads', 'uninative'), os.path.join(basepath, 'downloads'))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500213
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600214 if not sstate_mirrors:
215 with open(os.path.join(conf_dir, 'site.conf'), 'a') as f:
216 f.write('SCONF_VERSION = "%s"\n' % site_conf_version)
217 f.write('SSTATE_MIRRORS_append = " file://.* %s/sstate-cache/PATH \\n "\n' % updateserver)
218 finally:
219 shutil.rmtree(tmpsdk_dir)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500220
221 if not args.skip_prepare:
222 # Find all potentially updateable tasks
223 sdk_update_targets = []
224 tasks = ['do_populate_sysroot', 'do_packagedata']
225 for root, _, files in os.walk(stamps_dir):
226 for fn in files:
227 if not '.sigdata.' in fn:
228 for task in tasks:
229 if '.%s.' % task in fn or '.%s_setscene.' % task in fn:
230 sdk_update_targets.append('%s:%s' % (os.path.basename(root), task))
231 # Run bitbake command for the whole SDK
232 logger.info("Preparing build system... (This may take some time.)")
233 try:
234 exec_build_env_command(config.init_path, basepath, 'bitbake --setscene-only %s' % ' '.join(sdk_update_targets), stderr=subprocess.STDOUT)
235 output, _ = exec_build_env_command(config.init_path, basepath, 'bitbake -n %s' % ' '.join(sdk_update_targets), stderr=subprocess.STDOUT)
236 runlines = []
237 for line in output.splitlines():
238 if 'Running task ' in line:
239 runlines.append(line)
240 if runlines:
241 logger.error('Unexecuted tasks found in preparation log:\n %s' % '\n '.join(runlines))
242 return -1
243 except bb.process.ExecutionError as e:
244 logger.error('Preparation failed:\n%s' % e.stdout)
245 return -1
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500246 return 0
247
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500248def sdk_install(args, config, basepath, workspace):
249 """Entry point for the devtool sdk-install command"""
250
251 import oe.recipeutils
252 import bb.process
253
254 for recipe in args.recipename:
255 if recipe in workspace:
256 raise DevtoolError('recipe %s is a recipe in your workspace' % recipe)
257
258 tasks = ['do_populate_sysroot', 'do_packagedata']
259 stampprefixes = {}
260 def checkstamp(recipe):
261 stampprefix = stampprefixes[recipe]
262 stamps = glob.glob(stampprefix + '*')
263 for stamp in stamps:
264 if '.sigdata.' not in stamp and stamp.startswith((stampprefix + '.', stampprefix + '_setscene.')):
265 return True
266 else:
267 return False
268
269 install_recipes = []
270 tinfoil = setup_tinfoil(config_only=False, basepath=basepath)
271 try:
272 for recipe in args.recipename:
273 rd = parse_recipe(config, tinfoil, recipe, True)
274 if not rd:
275 return 1
276 stampprefixes[recipe] = '%s.%s' % (rd.getVar('STAMP', True), tasks[0])
277 if checkstamp(recipe):
278 logger.info('%s is already installed' % recipe)
279 else:
280 install_recipes.append(recipe)
281 finally:
282 tinfoil.shutdown()
283
284 if install_recipes:
285 logger.info('Installing %s...' % ', '.join(install_recipes))
286 install_tasks = []
287 for recipe in install_recipes:
288 for task in tasks:
289 if recipe.endswith('-native') and 'package' in task:
290 continue
291 install_tasks.append('%s:%s' % (recipe, task))
292 options = ''
293 if not args.allow_build:
294 options += ' --setscene-only'
295 try:
296 exec_build_env_command(config.init_path, basepath, 'bitbake %s %s' % (options, ' '.join(install_tasks)), watch=True)
297 except bb.process.ExecutionError as e:
298 raise DevtoolError('Failed to install %s:\n%s' % (recipe, str(e)))
299 failed = False
300 for recipe in install_recipes:
301 if checkstamp(recipe):
302 logger.info('Successfully installed %s' % recipe)
303 else:
304 raise DevtoolError('Failed to install %s - unavailable' % recipe)
305 failed = True
306 if failed:
307 return 2
308
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500309def register_commands(subparsers, context):
310 """Register devtool subcommands from the sdk plugin"""
311 if context.fixed_setup:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500312 parser_sdk = subparsers.add_parser('sdk-update',
313 help='Update SDK components',
314 description='Updates installed SDK components from a remote server',
315 group='sdk')
316 updateserver = context.config.get('SDK', 'updateserver', '')
317 if updateserver:
318 parser_sdk.add_argument('updateserver', help='The update server to fetch latest SDK components from (default %s)' % updateserver, nargs='?')
319 else:
320 parser_sdk.add_argument('updateserver', help='The update server to fetch latest SDK components from')
321 parser_sdk.add_argument('--skip-prepare', action="store_true", help='Skip re-preparing the build system after updating (for debugging only)')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500322 parser_sdk.set_defaults(func=sdk_update)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500323
324 parser_sdk_install = subparsers.add_parser('sdk-install',
325 help='Install additional SDK components',
326 description='Installs additional recipe development files into the SDK. (You can use "devtool search" to find available recipes.)',
327 group='sdk')
328 parser_sdk_install.add_argument('recipename', help='Name of the recipe to install the development artifacts for', nargs='+')
329 parser_sdk_install.add_argument('-s', '--allow-build', help='Allow building requested item(s) from source', action='store_true')
330 parser_sdk_install.set_defaults(func=sdk_install)