blob: e5af2c95aec1babffcae342786119c6423e9d2de [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# Development tool - deploy/undeploy command plugin
2#
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05003# Copyright (C) 2014-2016 Intel Corporation
Patrick Williamsc124f4f2015-09-15 14:41:29 -05004#
Brad Bishopc342db32019-05-15 21:57:59 -04005# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -05006#
Patrick Williamsc124f4f2015-09-15 14:41:29 -05007"""Devtool plugin containing the deploy subcommands"""
8
Patrick Williamsc124f4f2015-09-15 14:41:29 -05009import logging
Brad Bishopd7bf8c12018-02-25 22:55:05 -050010import os
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050011import shutil
Brad Bishopd7bf8c12018-02-25 22:55:05 -050012import subprocess
13import tempfile
14
15import bb.utils
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050016import argparse_oe
Brad Bishopd7bf8c12018-02-25 22:55:05 -050017import oe.types
18
Patrick Williamsf1e5d692016-03-30 15:21:19 -050019from devtool import exec_fakeroot, setup_tinfoil, check_workspace_recipe, DevtoolError
Patrick Williamsc124f4f2015-09-15 14:41:29 -050020
21logger = logging.getLogger('devtool')
22
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050023deploylist_path = '/.devtool'
24
25def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=False, nopreserve=False, nocheckspace=False):
26 """
27 Prepare a shell script for running on the target to
28 deploy/undeploy files. We have to be careful what we put in this
29 script - only commands that are likely to be available on the
30 target are suitable (the target might be constrained, e.g. using
31 busybox rather than bash with coreutils).
32 """
33 lines = []
34 lines.append('#!/bin/sh')
35 lines.append('set -e')
36 if undeployall:
37 # Yes, I know this is crude - but it does work
38 lines.append('for entry in %s/*.list; do' % deploylist_path)
39 lines.append('[ ! -f $entry ] && exit')
40 lines.append('set `basename $entry | sed "s/.list//"`')
41 if dryrun:
42 if not deploy:
43 lines.append('echo "Previously deployed files for $1:"')
44 lines.append('manifest="%s/$1.list"' % deploylist_path)
45 lines.append('preservedir="%s/$1.preserve"' % deploylist_path)
46 lines.append('if [ -f $manifest ] ; then')
47 # Read manifest in reverse and delete files / remove empty dirs
48 lines.append(' sed \'1!G;h;$!d\' $manifest | while read file')
49 lines.append(' do')
50 if dryrun:
51 lines.append(' if [ ! -d $file ] ; then')
52 lines.append(' echo $file')
53 lines.append(' fi')
54 else:
55 lines.append(' if [ -d $file ] ; then')
56 # Avoid deleting a preserved directory in case it has special perms
57 lines.append(' if [ ! -d $preservedir/$file ] ; then')
58 lines.append(' rmdir $file > /dev/null 2>&1 || true')
59 lines.append(' fi')
60 lines.append(' else')
Brad Bishopd7bf8c12018-02-25 22:55:05 -050061 lines.append(' rm -f $file')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050062 lines.append(' fi')
63 lines.append(' done')
64 if not dryrun:
65 lines.append(' rm $manifest')
66 if not deploy and not dryrun:
67 # May as well remove all traces
68 lines.append(' rmdir `dirname $manifest` > /dev/null 2>&1 || true')
69 lines.append('fi')
70
71 if deploy:
72 if not nocheckspace:
73 # Check for available space
74 # FIXME This doesn't take into account files spread across multiple
75 # partitions, but doing that is non-trivial
76 # Find the part of the destination path that exists
77 lines.append('checkpath="$2"')
78 lines.append('while [ "$checkpath" != "/" ] && [ ! -e $checkpath ]')
79 lines.append('do')
80 lines.append(' checkpath=`dirname "$checkpath"`')
81 lines.append('done')
Patrick Williamsc0f7c042017-02-23 20:41:17 -060082 lines.append(r'freespace=$(df -P $checkpath | sed -nre "s/^(\S+\s+){3}([0-9]+).*/\2/p")')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050083 # First line of the file is the total space
84 lines.append('total=`head -n1 $3`')
85 lines.append('if [ $total -gt $freespace ] ; then')
86 lines.append(' echo "ERROR: insufficient space on target (available ${freespace}, needed ${total})"')
87 lines.append(' exit 1')
88 lines.append('fi')
89 if not nopreserve:
90 # Preserve any files that exist. Note that this will add to the
91 # preserved list with successive deployments if the list of files
92 # deployed changes, but because we've deleted any previously
93 # deployed files at this point it will never preserve anything
94 # that was deployed, only files that existed prior to any deploying
95 # (which makes the most sense)
96 lines.append('cat $3 | sed "1d" | while read file fsize')
97 lines.append('do')
98 lines.append(' if [ -e $file ] ; then')
99 lines.append(' dest="$preservedir/$file"')
100 lines.append(' mkdir -p `dirname $dest`')
101 lines.append(' mv $file $dest')
102 lines.append(' fi')
103 lines.append('done')
104 lines.append('rm $3')
105 lines.append('mkdir -p `dirname $manifest`')
106 lines.append('mkdir -p $2')
107 if verbose:
108 lines.append(' tar xv -C $2 -f - | tee $manifest')
109 else:
110 lines.append(' tar xv -C $2 -f - > $manifest')
111 lines.append('sed -i "s!^./!$2!" $manifest')
112 elif not dryrun:
113 # Put any preserved files back
114 lines.append('if [ -d $preservedir ] ; then')
115 lines.append(' cd $preservedir')
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500116 # find from busybox might not have -exec, so we don't use that
117 lines.append(' find . -type f | while read file')
118 lines.append(' do')
119 lines.append(' mv $file /$file')
120 lines.append(' done')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500121 lines.append(' cd /')
122 lines.append(' rm -rf $preservedir')
123 lines.append('fi')
124
125 if undeployall:
126 if not dryrun:
127 lines.append('echo "NOTE: Successfully undeployed $1"')
128 lines.append('done')
129
130 # Delete the script itself
131 lines.append('rm $0')
132 lines.append('')
133
134 return '\n'.join(lines)
135
136
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500137
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500138def deploy(args, config, basepath, workspace):
139 """Entry point for the devtool 'deploy' subcommand"""
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500140 import math
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500141 import oe.recipeutils
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500142 import oe.package
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500143
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500144 check_workspace_recipe(workspace, args.recipename, checksrc=False)
145
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500146 try:
147 host, destdir = args.target.split(':')
148 except ValueError:
149 destdir = '/'
150 else:
151 args.target = host
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500152 if not destdir.endswith('/'):
153 destdir += '/'
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500154
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500155 tinfoil = setup_tinfoil(basepath=basepath)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500156 try:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600157 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500158 rd = tinfoil.parse_recipe(args.recipename)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600159 except Exception as e:
160 raise DevtoolError('Exception parsing recipe %s: %s' %
161 (args.recipename, e))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500162 recipe_outdir = rd.getVar('D')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600163 if not os.path.exists(recipe_outdir) or not os.listdir(recipe_outdir):
164 raise DevtoolError('No files to deploy - have you built the %s '
165 'recipe? If so, the install step has not installed '
166 'any files.' % args.recipename)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500167
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500168 if args.strip and not args.dry_run:
169 # Fakeroot copy to new destination
170 srcdir = recipe_outdir
171 recipe_outdir = os.path.join(rd.getVar('WORKDIR'), 'deploy-target-stripped')
172 if os.path.isdir(recipe_outdir):
173 bb.utils.remove(recipe_outdir, True)
174 exec_fakeroot(rd, "cp -af %s %s" % (os.path.join(srcdir, '.'), recipe_outdir), shell=True)
175 os.environ['PATH'] = ':'.join([os.environ['PATH'], rd.getVar('PATH') or ''])
176 oe.package.strip_execs(args.recipename, recipe_outdir, rd.getVar('STRIP'), rd.getVar('libdir'),
Brad Bishopa5c52ff2018-11-23 10:55:50 +1300177 rd.getVar('base_libdir'), rd)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500178
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600179 filelist = []
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500180 inodes = set({})
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600181 ftotalsize = 0
182 for root, _, files in os.walk(recipe_outdir):
183 for fn in files:
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500184 fstat = os.lstat(os.path.join(root, fn))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600185 # Get the size in kiB (since we'll be comparing it to the output of du -k)
186 # MUST use lstat() here not stat() or getfilesize() since we don't want to
187 # dereference symlinks
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500188 if fstat.st_ino in inodes:
189 fsize = 0
190 else:
191 fsize = int(math.ceil(float(fstat.st_size)/1024))
192 inodes.add(fstat.st_ino)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600193 ftotalsize += fsize
194 # The path as it would appear on the target
195 fpath = os.path.join(destdir, os.path.relpath(root, recipe_outdir), fn)
196 filelist.append((fpath, fsize))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500197
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600198 if args.dry_run:
199 print('Files to be deployed for %s on target %s:' % (args.recipename, args.target))
200 for item, _ in filelist:
201 print(' %s' % item)
202 return 0
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500203
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600204 extraoptions = ''
205 if args.no_host_check:
206 extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
207 if not args.show_status:
208 extraoptions += ' -q'
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500209
Brad Bishop19323692019-04-05 15:28:33 -0400210 scp_sshexec = ''
211 ssh_sshexec = 'ssh'
212 if args.ssh_exec:
213 scp_sshexec = "-S %s" % args.ssh_exec
214 ssh_sshexec = args.ssh_exec
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500215 scp_port = ''
216 ssh_port = ''
Brad Bishop316dfdd2018-06-25 12:45:53 -0400217 if args.port:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500218 scp_port = "-P %s" % args.port
219 ssh_port = "-p %s" % args.port
220
Brad Bishop64c979e2019-11-04 13:55:29 -0500221 if args.key:
222 extraoptions += ' -i %s' % args.key
223
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600224 # In order to delete previously deployed files and have the manifest file on
225 # the target, we write out a shell script and then copy it to the target
226 # so we can then run it (piping tar output to it).
227 # (We cannot use scp here, because it doesn't preserve symlinks.)
228 tmpdir = tempfile.mkdtemp(prefix='devtool')
229 try:
230 tmpscript = '/tmp/devtool_deploy.sh'
231 tmpfilelist = os.path.join(os.path.dirname(tmpscript), 'devtool_deploy.list')
232 shellscript = _prepare_remote_script(deploy=True,
233 verbose=args.show_status,
234 nopreserve=args.no_preserve,
235 nocheckspace=args.no_check_space)
236 # Write out the script to a file
237 with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f:
238 f.write(shellscript)
239 # Write out the file list
240 with open(os.path.join(tmpdir, os.path.basename(tmpfilelist)), 'w') as f:
241 f.write('%d\n' % ftotalsize)
242 for fpath, fsize in filelist:
243 f.write('%s %d\n' % (fpath, fsize))
244 # Copy them to the target
Brad Bishop19323692019-04-05 15:28:33 -0400245 ret = subprocess.call("scp %s %s %s %s/* %s:%s" % (scp_sshexec, scp_port, extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600246 if ret != 0:
247 raise DevtoolError('Failed to copy script to %s - rerun with -s to '
248 'get a complete error message' % args.target)
249 finally:
250 shutil.rmtree(tmpdir)
251
252 # Now run the script
Brad Bishop19323692019-04-05 15:28:33 -0400253 ret = exec_fakeroot(rd, 'tar cf - . | %s %s %s %s \'sh %s %s %s %s\'' % (ssh_sshexec, ssh_port, extraoptions, args.target, tmpscript, args.recipename, destdir, tmpfilelist), cwd=recipe_outdir, shell=True)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500254 if ret != 0:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600255 raise DevtoolError('Deploy failed - rerun with -s to get a complete '
256 'error message')
257
258 logger.info('Successfully deployed %s' % recipe_outdir)
259
260 files_list = []
261 for root, _, files in os.walk(recipe_outdir):
262 for filename in files:
263 filename = os.path.relpath(os.path.join(root, filename), recipe_outdir)
264 files_list.append(os.path.join(destdir, filename))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500265 finally:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600266 tinfoil.shutdown()
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500267
268 return 0
269
270def undeploy(args, config, basepath, workspace):
271 """Entry point for the devtool 'undeploy' subcommand"""
272 if args.all and args.recipename:
273 raise argparse_oe.ArgumentUsageError('Cannot specify -a/--all with a recipe name', 'undeploy-target')
274 elif not args.recipename and not args.all:
275 raise argparse_oe.ArgumentUsageError('If you don\'t specify a recipe, you must specify -a/--all', 'undeploy-target')
276
277 extraoptions = ''
278 if args.no_host_check:
279 extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
280 if not args.show_status:
281 extraoptions += ' -q'
282
Brad Bishop19323692019-04-05 15:28:33 -0400283 scp_sshexec = ''
284 ssh_sshexec = 'ssh'
285 if args.ssh_exec:
286 scp_sshexec = "-S %s" % args.ssh_exec
287 ssh_sshexec = args.ssh_exec
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500288 scp_port = ''
289 ssh_port = ''
Brad Bishop316dfdd2018-06-25 12:45:53 -0400290 if args.port:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500291 scp_port = "-P %s" % args.port
292 ssh_port = "-p %s" % args.port
293
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500294 args.target = args.target.split(':')[0]
295
296 tmpdir = tempfile.mkdtemp(prefix='devtool')
297 try:
298 tmpscript = '/tmp/devtool_undeploy.sh'
299 shellscript = _prepare_remote_script(deploy=False, dryrun=args.dry_run, undeployall=args.all)
300 # Write out the script to a file
301 with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f:
302 f.write(shellscript)
303 # Copy it to the target
Brad Bishop19323692019-04-05 15:28:33 -0400304 ret = subprocess.call("scp %s %s %s %s/* %s:%s" % (scp_sshexec, scp_port, extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500305 if ret != 0:
306 raise DevtoolError('Failed to copy script to %s - rerun with -s to '
307 'get a complete error message' % args.target)
308 finally:
309 shutil.rmtree(tmpdir)
310
311 # Now run the script
Brad Bishop19323692019-04-05 15:28:33 -0400312 ret = subprocess.call('%s %s %s %s \'sh %s %s\'' % (ssh_sshexec, ssh_port, extraoptions, args.target, tmpscript, args.recipename), shell=True)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500313 if ret != 0:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500314 raise DevtoolError('Undeploy failed - rerun with -s to get a complete '
315 'error message')
316
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500317 if not args.all and not args.dry_run:
318 logger.info('Successfully undeployed %s' % args.recipename)
319 return 0
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500320
321
322def register_commands(subparsers, context):
323 """Register devtool subcommands from the deploy plugin"""
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500324
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500325 parser_deploy = subparsers.add_parser('deploy-target',
326 help='Deploy recipe output files to live target machine',
327 description='Deploys a recipe\'s build output (i.e. the output of the do_install task) to a live target machine over ssh. By default, any existing files will be preserved instead of being overwritten and will be restored if you run devtool undeploy-target. Note: this only deploys the recipe itself and not any runtime dependencies, so it is assumed that those have been installed on the target beforehand.',
328 group='testbuild')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500329 parser_deploy.add_argument('recipename', help='Recipe to deploy')
330 parser_deploy.add_argument('target', help='Live target machine running an ssh server: user@hostname[:destdir]')
331 parser_deploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true')
332 parser_deploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true')
333 parser_deploy.add_argument('-n', '--dry-run', help='List files to be deployed only', action='store_true')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500334 parser_deploy.add_argument('-p', '--no-preserve', help='Do not preserve existing files', action='store_true')
335 parser_deploy.add_argument('--no-check-space', help='Do not check for available space before deploying', action='store_true')
Brad Bishop19323692019-04-05 15:28:33 -0400336 parser_deploy.add_argument('-e', '--ssh-exec', help='Executable to use in place of ssh')
Brad Bishop316dfdd2018-06-25 12:45:53 -0400337 parser_deploy.add_argument('-P', '--port', help='Specify port to use for connection to the target')
Brad Bishop64c979e2019-11-04 13:55:29 -0500338 parser_deploy.add_argument('-I', '--key',
Andrew Geisslerd25ed322020-06-27 00:28:28 -0500339 help='Specify ssh private key for connection to the target')
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500340
341 strip_opts = parser_deploy.add_mutually_exclusive_group(required=False)
342 strip_opts.add_argument('-S', '--strip',
343 help='Strip executables prior to deploying (default: %(default)s). '
344 'The default value of this option can be controlled by setting the strip option in the [Deploy] section to True or False.',
345 default=oe.types.boolean(context.config.get('Deploy', 'strip', default='0')),
346 action='store_true')
347 strip_opts.add_argument('--no-strip', help='Do not strip executables prior to deploy', dest='strip', action='store_false')
348
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500349 parser_deploy.set_defaults(func=deploy)
350
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500351 parser_undeploy = subparsers.add_parser('undeploy-target',
352 help='Undeploy recipe output files in live target machine',
353 description='Un-deploys recipe output files previously deployed to a live target machine by devtool deploy-target.',
354 group='testbuild')
355 parser_undeploy.add_argument('recipename', help='Recipe to undeploy (if not using -a/--all)', nargs='?')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500356 parser_undeploy.add_argument('target', help='Live target machine running an ssh server: user@hostname')
357 parser_undeploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true')
358 parser_undeploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500359 parser_undeploy.add_argument('-a', '--all', help='Undeploy all recipes deployed on the target', action='store_true')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500360 parser_undeploy.add_argument('-n', '--dry-run', help='List files to be undeployed only', action='store_true')
Brad Bishop19323692019-04-05 15:28:33 -0400361 parser_undeploy.add_argument('-e', '--ssh-exec', help='Executable to use in place of ssh')
Brad Bishop316dfdd2018-06-25 12:45:53 -0400362 parser_undeploy.add_argument('-P', '--port', help='Specify port to use for connection to the target')
Brad Bishop64c979e2019-11-04 13:55:29 -0500363 parser_undeploy.add_argument('-I', '--key',
Andrew Geisslerd25ed322020-06-27 00:28:28 -0500364 help='Specify ssh private key for connection to the target')
Brad Bishop64c979e2019-11-04 13:55:29 -0500365
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500366 parser_undeploy.set_defaults(func=undeploy)