blob: 6a997735fceeb513c7f9e20ba804da50d5c90c87 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# Development tool - deploy/undeploy command plugin
2#
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05003# Copyright (C) 2014-2016 Intel Corporation
Patrick Williamsc124f4f2015-09-15 14:41:29 -05004#
Brad Bishopc342db32019-05-15 21:57:59 -04005# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -05006#
Patrick Williamsc124f4f2015-09-15 14:41:29 -05007"""Devtool plugin containing the deploy subcommands"""
8
Patrick Williamsc124f4f2015-09-15 14:41:29 -05009import logging
Brad Bishopd7bf8c12018-02-25 22:55:05 -050010import os
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050011import shutil
Brad Bishopd7bf8c12018-02-25 22:55:05 -050012import subprocess
13import tempfile
14
15import bb.utils
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050016import argparse_oe
Brad Bishopd7bf8c12018-02-25 22:55:05 -050017import oe.types
18
Patrick Williamsf1e5d692016-03-30 15:21:19 -050019from devtool import exec_fakeroot, setup_tinfoil, check_workspace_recipe, DevtoolError
Patrick Williamsc124f4f2015-09-15 14:41:29 -050020
21logger = logging.getLogger('devtool')
22
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050023deploylist_path = '/.devtool'
24
25def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=False, nopreserve=False, nocheckspace=False):
26 """
27 Prepare a shell script for running on the target to
28 deploy/undeploy files. We have to be careful what we put in this
29 script - only commands that are likely to be available on the
30 target are suitable (the target might be constrained, e.g. using
31 busybox rather than bash with coreutils).
32 """
33 lines = []
34 lines.append('#!/bin/sh')
35 lines.append('set -e')
36 if undeployall:
37 # Yes, I know this is crude - but it does work
38 lines.append('for entry in %s/*.list; do' % deploylist_path)
39 lines.append('[ ! -f $entry ] && exit')
40 lines.append('set `basename $entry | sed "s/.list//"`')
41 if dryrun:
42 if not deploy:
43 lines.append('echo "Previously deployed files for $1:"')
44 lines.append('manifest="%s/$1.list"' % deploylist_path)
45 lines.append('preservedir="%s/$1.preserve"' % deploylist_path)
46 lines.append('if [ -f $manifest ] ; then')
47 # Read manifest in reverse and delete files / remove empty dirs
48 lines.append(' sed \'1!G;h;$!d\' $manifest | while read file')
49 lines.append(' do')
50 if dryrun:
51 lines.append(' if [ ! -d $file ] ; then')
52 lines.append(' echo $file')
53 lines.append(' fi')
54 else:
55 lines.append(' if [ -d $file ] ; then')
56 # Avoid deleting a preserved directory in case it has special perms
57 lines.append(' if [ ! -d $preservedir/$file ] ; then')
58 lines.append(' rmdir $file > /dev/null 2>&1 || true')
59 lines.append(' fi')
60 lines.append(' else')
Brad Bishopd7bf8c12018-02-25 22:55:05 -050061 lines.append(' rm -f $file')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050062 lines.append(' fi')
63 lines.append(' done')
64 if not dryrun:
65 lines.append(' rm $manifest')
66 if not deploy and not dryrun:
67 # May as well remove all traces
68 lines.append(' rmdir `dirname $manifest` > /dev/null 2>&1 || true')
69 lines.append('fi')
70
71 if deploy:
72 if not nocheckspace:
73 # Check for available space
74 # FIXME This doesn't take into account files spread across multiple
75 # partitions, but doing that is non-trivial
76 # Find the part of the destination path that exists
77 lines.append('checkpath="$2"')
78 lines.append('while [ "$checkpath" != "/" ] && [ ! -e $checkpath ]')
79 lines.append('do')
80 lines.append(' checkpath=`dirname "$checkpath"`')
81 lines.append('done')
Patrick Williamsc0f7c042017-02-23 20:41:17 -060082 lines.append(r'freespace=$(df -P $checkpath | sed -nre "s/^(\S+\s+){3}([0-9]+).*/\2/p")')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050083 # First line of the file is the total space
84 lines.append('total=`head -n1 $3`')
85 lines.append('if [ $total -gt $freespace ] ; then')
86 lines.append(' echo "ERROR: insufficient space on target (available ${freespace}, needed ${total})"')
87 lines.append(' exit 1')
88 lines.append('fi')
89 if not nopreserve:
90 # Preserve any files that exist. Note that this will add to the
91 # preserved list with successive deployments if the list of files
92 # deployed changes, but because we've deleted any previously
93 # deployed files at this point it will never preserve anything
94 # that was deployed, only files that existed prior to any deploying
95 # (which makes the most sense)
96 lines.append('cat $3 | sed "1d" | while read file fsize')
97 lines.append('do')
98 lines.append(' if [ -e $file ] ; then')
99 lines.append(' dest="$preservedir/$file"')
100 lines.append(' mkdir -p `dirname $dest`')
101 lines.append(' mv $file $dest')
102 lines.append(' fi')
103 lines.append('done')
104 lines.append('rm $3')
105 lines.append('mkdir -p `dirname $manifest`')
106 lines.append('mkdir -p $2')
107 if verbose:
108 lines.append(' tar xv -C $2 -f - | tee $manifest')
109 else:
110 lines.append(' tar xv -C $2 -f - > $manifest')
111 lines.append('sed -i "s!^./!$2!" $manifest')
112 elif not dryrun:
113 # Put any preserved files back
114 lines.append('if [ -d $preservedir ] ; then')
115 lines.append(' cd $preservedir')
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500116 # find from busybox might not have -exec, so we don't use that
117 lines.append(' find . -type f | while read file')
118 lines.append(' do')
119 lines.append(' mv $file /$file')
120 lines.append(' done')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500121 lines.append(' cd /')
122 lines.append(' rm -rf $preservedir')
123 lines.append('fi')
124
125 if undeployall:
126 if not dryrun:
127 lines.append('echo "NOTE: Successfully undeployed $1"')
128 lines.append('done')
129
130 # Delete the script itself
131 lines.append('rm $0')
132 lines.append('')
133
134 return '\n'.join(lines)
135
136
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500137
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500138def deploy(args, config, basepath, workspace):
139 """Entry point for the devtool 'deploy' subcommand"""
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500140 import math
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500141 import oe.recipeutils
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500142 import oe.package
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500143
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500144 check_workspace_recipe(workspace, args.recipename, checksrc=False)
145
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500146 try:
147 host, destdir = args.target.split(':')
148 except ValueError:
149 destdir = '/'
150 else:
151 args.target = host
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500152 if not destdir.endswith('/'):
153 destdir += '/'
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500154
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500155 tinfoil = setup_tinfoil(basepath=basepath)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500156 try:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600157 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500158 rd = tinfoil.parse_recipe(args.recipename)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600159 except Exception as e:
160 raise DevtoolError('Exception parsing recipe %s: %s' %
161 (args.recipename, e))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500162 recipe_outdir = rd.getVar('D')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600163 if not os.path.exists(recipe_outdir) or not os.listdir(recipe_outdir):
164 raise DevtoolError('No files to deploy - have you built the %s '
165 'recipe? If so, the install step has not installed '
166 'any files.' % args.recipename)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500167
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500168 if args.strip and not args.dry_run:
169 # Fakeroot copy to new destination
170 srcdir = recipe_outdir
171 recipe_outdir = os.path.join(rd.getVar('WORKDIR'), 'deploy-target-stripped')
172 if os.path.isdir(recipe_outdir):
173 bb.utils.remove(recipe_outdir, True)
174 exec_fakeroot(rd, "cp -af %s %s" % (os.path.join(srcdir, '.'), recipe_outdir), shell=True)
175 os.environ['PATH'] = ':'.join([os.environ['PATH'], rd.getVar('PATH') or ''])
176 oe.package.strip_execs(args.recipename, recipe_outdir, rd.getVar('STRIP'), rd.getVar('libdir'),
Brad Bishopa5c52ff2018-11-23 10:55:50 +1300177 rd.getVar('base_libdir'), rd)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500178
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600179 filelist = []
180 ftotalsize = 0
181 for root, _, files in os.walk(recipe_outdir):
182 for fn in files:
183 # Get the size in kiB (since we'll be comparing it to the output of du -k)
184 # MUST use lstat() here not stat() or getfilesize() since we don't want to
185 # dereference symlinks
186 fsize = int(math.ceil(float(os.lstat(os.path.join(root, fn)).st_size)/1024))
187 ftotalsize += fsize
188 # The path as it would appear on the target
189 fpath = os.path.join(destdir, os.path.relpath(root, recipe_outdir), fn)
190 filelist.append((fpath, fsize))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500191
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600192 if args.dry_run:
193 print('Files to be deployed for %s on target %s:' % (args.recipename, args.target))
194 for item, _ in filelist:
195 print(' %s' % item)
196 return 0
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500197
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600198 extraoptions = ''
199 if args.no_host_check:
200 extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
201 if not args.show_status:
202 extraoptions += ' -q'
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500203
Brad Bishop19323692019-04-05 15:28:33 -0400204 scp_sshexec = ''
205 ssh_sshexec = 'ssh'
206 if args.ssh_exec:
207 scp_sshexec = "-S %s" % args.ssh_exec
208 ssh_sshexec = args.ssh_exec
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500209 scp_port = ''
210 ssh_port = ''
Brad Bishop316dfdd2018-06-25 12:45:53 -0400211 if args.port:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500212 scp_port = "-P %s" % args.port
213 ssh_port = "-p %s" % args.port
214
Brad Bishop64c979e2019-11-04 13:55:29 -0500215 if args.key:
216 extraoptions += ' -i %s' % args.key
217
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600218 # In order to delete previously deployed files and have the manifest file on
219 # the target, we write out a shell script and then copy it to the target
220 # so we can then run it (piping tar output to it).
221 # (We cannot use scp here, because it doesn't preserve symlinks.)
222 tmpdir = tempfile.mkdtemp(prefix='devtool')
223 try:
224 tmpscript = '/tmp/devtool_deploy.sh'
225 tmpfilelist = os.path.join(os.path.dirname(tmpscript), 'devtool_deploy.list')
226 shellscript = _prepare_remote_script(deploy=True,
227 verbose=args.show_status,
228 nopreserve=args.no_preserve,
229 nocheckspace=args.no_check_space)
230 # Write out the script to a file
231 with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f:
232 f.write(shellscript)
233 # Write out the file list
234 with open(os.path.join(tmpdir, os.path.basename(tmpfilelist)), 'w') as f:
235 f.write('%d\n' % ftotalsize)
236 for fpath, fsize in filelist:
237 f.write('%s %d\n' % (fpath, fsize))
238 # Copy them to the target
Brad Bishop19323692019-04-05 15:28:33 -0400239 ret = subprocess.call("scp %s %s %s %s/* %s:%s" % (scp_sshexec, scp_port, extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600240 if ret != 0:
241 raise DevtoolError('Failed to copy script to %s - rerun with -s to '
242 'get a complete error message' % args.target)
243 finally:
244 shutil.rmtree(tmpdir)
245
246 # Now run the script
Brad Bishop19323692019-04-05 15:28:33 -0400247 ret = exec_fakeroot(rd, 'tar cf - . | %s %s %s %s \'sh %s %s %s %s\'' % (ssh_sshexec, ssh_port, extraoptions, args.target, tmpscript, args.recipename, destdir, tmpfilelist), cwd=recipe_outdir, shell=True)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500248 if ret != 0:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600249 raise DevtoolError('Deploy failed - rerun with -s to get a complete '
250 'error message')
251
252 logger.info('Successfully deployed %s' % recipe_outdir)
253
254 files_list = []
255 for root, _, files in os.walk(recipe_outdir):
256 for filename in files:
257 filename = os.path.relpath(os.path.join(root, filename), recipe_outdir)
258 files_list.append(os.path.join(destdir, filename))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500259 finally:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600260 tinfoil.shutdown()
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500261
262 return 0
263
264def undeploy(args, config, basepath, workspace):
265 """Entry point for the devtool 'undeploy' subcommand"""
266 if args.all and args.recipename:
267 raise argparse_oe.ArgumentUsageError('Cannot specify -a/--all with a recipe name', 'undeploy-target')
268 elif not args.recipename and not args.all:
269 raise argparse_oe.ArgumentUsageError('If you don\'t specify a recipe, you must specify -a/--all', 'undeploy-target')
270
271 extraoptions = ''
272 if args.no_host_check:
273 extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
274 if not args.show_status:
275 extraoptions += ' -q'
276
Brad Bishop19323692019-04-05 15:28:33 -0400277 scp_sshexec = ''
278 ssh_sshexec = 'ssh'
279 if args.ssh_exec:
280 scp_sshexec = "-S %s" % args.ssh_exec
281 ssh_sshexec = args.ssh_exec
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500282 scp_port = ''
283 ssh_port = ''
Brad Bishop316dfdd2018-06-25 12:45:53 -0400284 if args.port:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500285 scp_port = "-P %s" % args.port
286 ssh_port = "-p %s" % args.port
287
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500288 args.target = args.target.split(':')[0]
289
290 tmpdir = tempfile.mkdtemp(prefix='devtool')
291 try:
292 tmpscript = '/tmp/devtool_undeploy.sh'
293 shellscript = _prepare_remote_script(deploy=False, dryrun=args.dry_run, undeployall=args.all)
294 # Write out the script to a file
295 with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f:
296 f.write(shellscript)
297 # Copy it to the target
Brad Bishop19323692019-04-05 15:28:33 -0400298 ret = subprocess.call("scp %s %s %s %s/* %s:%s" % (scp_sshexec, scp_port, extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500299 if ret != 0:
300 raise DevtoolError('Failed to copy script to %s - rerun with -s to '
301 'get a complete error message' % args.target)
302 finally:
303 shutil.rmtree(tmpdir)
304
305 # Now run the script
Brad Bishop19323692019-04-05 15:28:33 -0400306 ret = subprocess.call('%s %s %s %s \'sh %s %s\'' % (ssh_sshexec, ssh_port, extraoptions, args.target, tmpscript, args.recipename), shell=True)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500307 if ret != 0:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500308 raise DevtoolError('Undeploy failed - rerun with -s to get a complete '
309 'error message')
310
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500311 if not args.all and not args.dry_run:
312 logger.info('Successfully undeployed %s' % args.recipename)
313 return 0
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500314
315
316def register_commands(subparsers, context):
317 """Register devtool subcommands from the deploy plugin"""
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500318
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500319 parser_deploy = subparsers.add_parser('deploy-target',
320 help='Deploy recipe output files to live target machine',
321 description='Deploys a recipe\'s build output (i.e. the output of the do_install task) to a live target machine over ssh. By default, any existing files will be preserved instead of being overwritten and will be restored if you run devtool undeploy-target. Note: this only deploys the recipe itself and not any runtime dependencies, so it is assumed that those have been installed on the target beforehand.',
322 group='testbuild')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500323 parser_deploy.add_argument('recipename', help='Recipe to deploy')
324 parser_deploy.add_argument('target', help='Live target machine running an ssh server: user@hostname[:destdir]')
325 parser_deploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true')
326 parser_deploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true')
327 parser_deploy.add_argument('-n', '--dry-run', help='List files to be deployed only', action='store_true')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500328 parser_deploy.add_argument('-p', '--no-preserve', help='Do not preserve existing files', action='store_true')
329 parser_deploy.add_argument('--no-check-space', help='Do not check for available space before deploying', action='store_true')
Brad Bishop19323692019-04-05 15:28:33 -0400330 parser_deploy.add_argument('-e', '--ssh-exec', help='Executable to use in place of ssh')
Brad Bishop316dfdd2018-06-25 12:45:53 -0400331 parser_deploy.add_argument('-P', '--port', help='Specify port to use for connection to the target')
Brad Bishop64c979e2019-11-04 13:55:29 -0500332 parser_deploy.add_argument('-I', '--key',
333 help='Specifiy ssh private key for connection to the target')
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500334
335 strip_opts = parser_deploy.add_mutually_exclusive_group(required=False)
336 strip_opts.add_argument('-S', '--strip',
337 help='Strip executables prior to deploying (default: %(default)s). '
338 'The default value of this option can be controlled by setting the strip option in the [Deploy] section to True or False.',
339 default=oe.types.boolean(context.config.get('Deploy', 'strip', default='0')),
340 action='store_true')
341 strip_opts.add_argument('--no-strip', help='Do not strip executables prior to deploy', dest='strip', action='store_false')
342
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500343 parser_deploy.set_defaults(func=deploy)
344
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500345 parser_undeploy = subparsers.add_parser('undeploy-target',
346 help='Undeploy recipe output files in live target machine',
347 description='Un-deploys recipe output files previously deployed to a live target machine by devtool deploy-target.',
348 group='testbuild')
349 parser_undeploy.add_argument('recipename', help='Recipe to undeploy (if not using -a/--all)', nargs='?')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500350 parser_undeploy.add_argument('target', help='Live target machine running an ssh server: user@hostname')
351 parser_undeploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true')
352 parser_undeploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500353 parser_undeploy.add_argument('-a', '--all', help='Undeploy all recipes deployed on the target', action='store_true')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500354 parser_undeploy.add_argument('-n', '--dry-run', help='List files to be undeployed only', action='store_true')
Brad Bishop19323692019-04-05 15:28:33 -0400355 parser_undeploy.add_argument('-e', '--ssh-exec', help='Executable to use in place of ssh')
Brad Bishop316dfdd2018-06-25 12:45:53 -0400356 parser_undeploy.add_argument('-P', '--port', help='Specify port to use for connection to the target')
Brad Bishop64c979e2019-11-04 13:55:29 -0500357 parser_undeploy.add_argument('-I', '--key',
358 help='Specifiy ssh private key for connection to the target')
359
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500360 parser_undeploy.set_defaults(func=undeploy)