blob: 886004b5d0cbfdb871959b5521c8718452f6b293 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# Development tool - deploy/undeploy command plugin
2#
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05003# Copyright (C) 2014-2016 Intel Corporation
Patrick Williamsc124f4f2015-09-15 14:41:29 -05004#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License version 2 as
7# published by the Free Software Foundation.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License along
15# with this program; if not, write to the Free Software Foundation, Inc.,
16# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
17"""Devtool plugin containing the deploy subcommands"""
18
Patrick Williamsc124f4f2015-09-15 14:41:29 -050019import logging
Brad Bishopd7bf8c12018-02-25 22:55:05 -050020import os
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050021import shutil
Brad Bishopd7bf8c12018-02-25 22:55:05 -050022import subprocess
23import tempfile
24
25import bb.utils
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050026import argparse_oe
Brad Bishopd7bf8c12018-02-25 22:55:05 -050027import oe.types
28
Patrick Williamsf1e5d692016-03-30 15:21:19 -050029from devtool import exec_fakeroot, setup_tinfoil, check_workspace_recipe, DevtoolError
Patrick Williamsc124f4f2015-09-15 14:41:29 -050030
31logger = logging.getLogger('devtool')
32
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050033deploylist_path = '/.devtool'
34
35def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=False, nopreserve=False, nocheckspace=False):
36 """
37 Prepare a shell script for running on the target to
38 deploy/undeploy files. We have to be careful what we put in this
39 script - only commands that are likely to be available on the
40 target are suitable (the target might be constrained, e.g. using
41 busybox rather than bash with coreutils).
42 """
43 lines = []
44 lines.append('#!/bin/sh')
45 lines.append('set -e')
46 if undeployall:
47 # Yes, I know this is crude - but it does work
48 lines.append('for entry in %s/*.list; do' % deploylist_path)
49 lines.append('[ ! -f $entry ] && exit')
50 lines.append('set `basename $entry | sed "s/.list//"`')
51 if dryrun:
52 if not deploy:
53 lines.append('echo "Previously deployed files for $1:"')
54 lines.append('manifest="%s/$1.list"' % deploylist_path)
55 lines.append('preservedir="%s/$1.preserve"' % deploylist_path)
56 lines.append('if [ -f $manifest ] ; then')
57 # Read manifest in reverse and delete files / remove empty dirs
58 lines.append(' sed \'1!G;h;$!d\' $manifest | while read file')
59 lines.append(' do')
60 if dryrun:
61 lines.append(' if [ ! -d $file ] ; then')
62 lines.append(' echo $file')
63 lines.append(' fi')
64 else:
65 lines.append(' if [ -d $file ] ; then')
66 # Avoid deleting a preserved directory in case it has special perms
67 lines.append(' if [ ! -d $preservedir/$file ] ; then')
68 lines.append(' rmdir $file > /dev/null 2>&1 || true')
69 lines.append(' fi')
70 lines.append(' else')
Brad Bishopd7bf8c12018-02-25 22:55:05 -050071 lines.append(' rm -f $file')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050072 lines.append(' fi')
73 lines.append(' done')
74 if not dryrun:
75 lines.append(' rm $manifest')
76 if not deploy and not dryrun:
77 # May as well remove all traces
78 lines.append(' rmdir `dirname $manifest` > /dev/null 2>&1 || true')
79 lines.append('fi')
80
81 if deploy:
82 if not nocheckspace:
83 # Check for available space
84 # FIXME This doesn't take into account files spread across multiple
85 # partitions, but doing that is non-trivial
86 # Find the part of the destination path that exists
87 lines.append('checkpath="$2"')
88 lines.append('while [ "$checkpath" != "/" ] && [ ! -e $checkpath ]')
89 lines.append('do')
90 lines.append(' checkpath=`dirname "$checkpath"`')
91 lines.append('done')
Patrick Williamsc0f7c042017-02-23 20:41:17 -060092 lines.append(r'freespace=$(df -P $checkpath | sed -nre "s/^(\S+\s+){3}([0-9]+).*/\2/p")')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050093 # First line of the file is the total space
94 lines.append('total=`head -n1 $3`')
95 lines.append('if [ $total -gt $freespace ] ; then')
96 lines.append(' echo "ERROR: insufficient space on target (available ${freespace}, needed ${total})"')
97 lines.append(' exit 1')
98 lines.append('fi')
99 if not nopreserve:
100 # Preserve any files that exist. Note that this will add to the
101 # preserved list with successive deployments if the list of files
102 # deployed changes, but because we've deleted any previously
103 # deployed files at this point it will never preserve anything
104 # that was deployed, only files that existed prior to any deploying
105 # (which makes the most sense)
106 lines.append('cat $3 | sed "1d" | while read file fsize')
107 lines.append('do')
108 lines.append(' if [ -e $file ] ; then')
109 lines.append(' dest="$preservedir/$file"')
110 lines.append(' mkdir -p `dirname $dest`')
111 lines.append(' mv $file $dest')
112 lines.append(' fi')
113 lines.append('done')
114 lines.append('rm $3')
115 lines.append('mkdir -p `dirname $manifest`')
116 lines.append('mkdir -p $2')
117 if verbose:
118 lines.append(' tar xv -C $2 -f - | tee $manifest')
119 else:
120 lines.append(' tar xv -C $2 -f - > $manifest')
121 lines.append('sed -i "s!^./!$2!" $manifest')
122 elif not dryrun:
123 # Put any preserved files back
124 lines.append('if [ -d $preservedir ] ; then')
125 lines.append(' cd $preservedir')
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500126 # find from busybox might not have -exec, so we don't use that
127 lines.append(' find . -type f | while read file')
128 lines.append(' do')
129 lines.append(' mv $file /$file')
130 lines.append(' done')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500131 lines.append(' cd /')
132 lines.append(' rm -rf $preservedir')
133 lines.append('fi')
134
135 if undeployall:
136 if not dryrun:
137 lines.append('echo "NOTE: Successfully undeployed $1"')
138 lines.append('done')
139
140 # Delete the script itself
141 lines.append('rm $0')
142 lines.append('')
143
144 return '\n'.join(lines)
145
146
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500147
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500148def deploy(args, config, basepath, workspace):
149 """Entry point for the devtool 'deploy' subcommand"""
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500150 import math
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500151 import oe.recipeutils
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500152 import oe.package
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500153
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500154 check_workspace_recipe(workspace, args.recipename, checksrc=False)
155
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500156 try:
157 host, destdir = args.target.split(':')
158 except ValueError:
159 destdir = '/'
160 else:
161 args.target = host
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500162 if not destdir.endswith('/'):
163 destdir += '/'
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500164
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500165 tinfoil = setup_tinfoil(basepath=basepath)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500166 try:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600167 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500168 rd = tinfoil.parse_recipe(args.recipename)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600169 except Exception as e:
170 raise DevtoolError('Exception parsing recipe %s: %s' %
171 (args.recipename, e))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500172 recipe_outdir = rd.getVar('D')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600173 if not os.path.exists(recipe_outdir) or not os.listdir(recipe_outdir):
174 raise DevtoolError('No files to deploy - have you built the %s '
175 'recipe? If so, the install step has not installed '
176 'any files.' % args.recipename)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500177
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500178 if args.strip and not args.dry_run:
179 # Fakeroot copy to new destination
180 srcdir = recipe_outdir
181 recipe_outdir = os.path.join(rd.getVar('WORKDIR'), 'deploy-target-stripped')
182 if os.path.isdir(recipe_outdir):
183 bb.utils.remove(recipe_outdir, True)
184 exec_fakeroot(rd, "cp -af %s %s" % (os.path.join(srcdir, '.'), recipe_outdir), shell=True)
185 os.environ['PATH'] = ':'.join([os.environ['PATH'], rd.getVar('PATH') or ''])
186 oe.package.strip_execs(args.recipename, recipe_outdir, rd.getVar('STRIP'), rd.getVar('libdir'),
Brad Bishopa5c52ff2018-11-23 10:55:50 +1300187 rd.getVar('base_libdir'), rd)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500188
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600189 filelist = []
190 ftotalsize = 0
191 for root, _, files in os.walk(recipe_outdir):
192 for fn in files:
193 # Get the size in kiB (since we'll be comparing it to the output of du -k)
194 # MUST use lstat() here not stat() or getfilesize() since we don't want to
195 # dereference symlinks
196 fsize = int(math.ceil(float(os.lstat(os.path.join(root, fn)).st_size)/1024))
197 ftotalsize += fsize
198 # The path as it would appear on the target
199 fpath = os.path.join(destdir, os.path.relpath(root, recipe_outdir), fn)
200 filelist.append((fpath, fsize))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500201
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600202 if args.dry_run:
203 print('Files to be deployed for %s on target %s:' % (args.recipename, args.target))
204 for item, _ in filelist:
205 print(' %s' % item)
206 return 0
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500207
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600208 extraoptions = ''
209 if args.no_host_check:
210 extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
211 if not args.show_status:
212 extraoptions += ' -q'
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500213
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500214 scp_port = ''
215 ssh_port = ''
Brad Bishop316dfdd2018-06-25 12:45:53 -0400216 if args.port:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500217 scp_port = "-P %s" % args.port
218 ssh_port = "-p %s" % args.port
219
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600220 # In order to delete previously deployed files and have the manifest file on
221 # the target, we write out a shell script and then copy it to the target
222 # so we can then run it (piping tar output to it).
223 # (We cannot use scp here, because it doesn't preserve symlinks.)
224 tmpdir = tempfile.mkdtemp(prefix='devtool')
225 try:
226 tmpscript = '/tmp/devtool_deploy.sh'
227 tmpfilelist = os.path.join(os.path.dirname(tmpscript), 'devtool_deploy.list')
228 shellscript = _prepare_remote_script(deploy=True,
229 verbose=args.show_status,
230 nopreserve=args.no_preserve,
231 nocheckspace=args.no_check_space)
232 # Write out the script to a file
233 with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f:
234 f.write(shellscript)
235 # Write out the file list
236 with open(os.path.join(tmpdir, os.path.basename(tmpfilelist)), 'w') as f:
237 f.write('%d\n' % ftotalsize)
238 for fpath, fsize in filelist:
239 f.write('%s %d\n' % (fpath, fsize))
240 # Copy them to the target
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500241 ret = subprocess.call("scp %s %s %s/* %s:%s" % (scp_port, extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600242 if ret != 0:
243 raise DevtoolError('Failed to copy script to %s - rerun with -s to '
244 'get a complete error message' % args.target)
245 finally:
246 shutil.rmtree(tmpdir)
247
248 # Now run the script
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500249 ret = exec_fakeroot(rd, 'tar cf - . | ssh %s %s %s \'sh %s %s %s %s\'' % (ssh_port, extraoptions, args.target, tmpscript, args.recipename, destdir, tmpfilelist), cwd=recipe_outdir, shell=True)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500250 if ret != 0:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600251 raise DevtoolError('Deploy failed - rerun with -s to get a complete '
252 'error message')
253
254 logger.info('Successfully deployed %s' % recipe_outdir)
255
256 files_list = []
257 for root, _, files in os.walk(recipe_outdir):
258 for filename in files:
259 filename = os.path.relpath(os.path.join(root, filename), recipe_outdir)
260 files_list.append(os.path.join(destdir, filename))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500261 finally:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600262 tinfoil.shutdown()
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500263
264 return 0
265
266def undeploy(args, config, basepath, workspace):
267 """Entry point for the devtool 'undeploy' subcommand"""
268 if args.all and args.recipename:
269 raise argparse_oe.ArgumentUsageError('Cannot specify -a/--all with a recipe name', 'undeploy-target')
270 elif not args.recipename and not args.all:
271 raise argparse_oe.ArgumentUsageError('If you don\'t specify a recipe, you must specify -a/--all', 'undeploy-target')
272
273 extraoptions = ''
274 if args.no_host_check:
275 extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
276 if not args.show_status:
277 extraoptions += ' -q'
278
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500279 scp_port = ''
280 ssh_port = ''
Brad Bishop316dfdd2018-06-25 12:45:53 -0400281 if args.port:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500282 scp_port = "-P %s" % args.port
283 ssh_port = "-p %s" % args.port
284
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500285 args.target = args.target.split(':')[0]
286
287 tmpdir = tempfile.mkdtemp(prefix='devtool')
288 try:
289 tmpscript = '/tmp/devtool_undeploy.sh'
290 shellscript = _prepare_remote_script(deploy=False, dryrun=args.dry_run, undeployall=args.all)
291 # Write out the script to a file
292 with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f:
293 f.write(shellscript)
294 # Copy it to the target
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500295 ret = subprocess.call("scp %s %s %s/* %s:%s" % (scp_port, extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500296 if ret != 0:
297 raise DevtoolError('Failed to copy script to %s - rerun with -s to '
298 'get a complete error message' % args.target)
299 finally:
300 shutil.rmtree(tmpdir)
301
302 # Now run the script
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500303 ret = subprocess.call('ssh %s %s %s \'sh %s %s\'' % (ssh_port, extraoptions, args.target, tmpscript, args.recipename), shell=True)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500304 if ret != 0:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500305 raise DevtoolError('Undeploy failed - rerun with -s to get a complete '
306 'error message')
307
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500308 if not args.all and not args.dry_run:
309 logger.info('Successfully undeployed %s' % args.recipename)
310 return 0
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500311
312
313def register_commands(subparsers, context):
314 """Register devtool subcommands from the deploy plugin"""
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500315
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500316 parser_deploy = subparsers.add_parser('deploy-target',
317 help='Deploy recipe output files to live target machine',
318 description='Deploys a recipe\'s build output (i.e. the output of the do_install task) to a live target machine over ssh. By default, any existing files will be preserved instead of being overwritten and will be restored if you run devtool undeploy-target. Note: this only deploys the recipe itself and not any runtime dependencies, so it is assumed that those have been installed on the target beforehand.',
319 group='testbuild')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500320 parser_deploy.add_argument('recipename', help='Recipe to deploy')
321 parser_deploy.add_argument('target', help='Live target machine running an ssh server: user@hostname[:destdir]')
322 parser_deploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true')
323 parser_deploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true')
324 parser_deploy.add_argument('-n', '--dry-run', help='List files to be deployed only', action='store_true')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500325 parser_deploy.add_argument('-p', '--no-preserve', help='Do not preserve existing files', action='store_true')
326 parser_deploy.add_argument('--no-check-space', help='Do not check for available space before deploying', action='store_true')
Brad Bishop316dfdd2018-06-25 12:45:53 -0400327 parser_deploy.add_argument('-P', '--port', help='Specify port to use for connection to the target')
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500328
329 strip_opts = parser_deploy.add_mutually_exclusive_group(required=False)
330 strip_opts.add_argument('-S', '--strip',
331 help='Strip executables prior to deploying (default: %(default)s). '
332 'The default value of this option can be controlled by setting the strip option in the [Deploy] section to True or False.',
333 default=oe.types.boolean(context.config.get('Deploy', 'strip', default='0')),
334 action='store_true')
335 strip_opts.add_argument('--no-strip', help='Do not strip executables prior to deploy', dest='strip', action='store_false')
336
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500337 parser_deploy.set_defaults(func=deploy)
338
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500339 parser_undeploy = subparsers.add_parser('undeploy-target',
340 help='Undeploy recipe output files in live target machine',
341 description='Un-deploys recipe output files previously deployed to a live target machine by devtool deploy-target.',
342 group='testbuild')
343 parser_undeploy.add_argument('recipename', help='Recipe to undeploy (if not using -a/--all)', nargs='?')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500344 parser_undeploy.add_argument('target', help='Live target machine running an ssh server: user@hostname')
345 parser_undeploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true')
346 parser_undeploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500347 parser_undeploy.add_argument('-a', '--all', help='Undeploy all recipes deployed on the target', action='store_true')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500348 parser_undeploy.add_argument('-n', '--dry-run', help='List files to be undeployed only', action='store_true')
Brad Bishop316dfdd2018-06-25 12:45:53 -0400349 parser_undeploy.add_argument('-P', '--port', help='Specify port to use for connection to the target')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500350 parser_undeploy.set_defaults(func=undeploy)