blob: 5ff1e230fddd9d06a8fcc0db14940f45b1d62152 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# Development tool - standard commands plugin
2#
Patrick Williamsc0f7c042017-02-23 20:41:17 -06003# Copyright (C) 2014-2016 Intel Corporation
Patrick Williamsc124f4f2015-09-15 14:41:29 -05004#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License version 2 as
7# published by the Free Software Foundation.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License along
15# with this program; if not, write to the Free Software Foundation, Inc.,
16# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
17"""Devtool standard plugins"""
18
19import os
20import sys
21import re
22import shutil
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050023import subprocess
Patrick Williamsc124f4f2015-09-15 14:41:29 -050024import tempfile
25import logging
26import argparse
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050027import argparse_oe
Patrick Williamsc124f4f2015-09-15 14:41:29 -050028import scriptutils
29import errno
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050030import glob
Patrick Williamsc0f7c042017-02-23 20:41:17 -060031import filecmp
Patrick Williamsf1e5d692016-03-30 15:21:19 -050032from collections import OrderedDict
Patrick Williamsc0f7c042017-02-23 20:41:17 -060033from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, use_external_build, setup_git_repo, recipe_to_append, get_bbclassextend_targets, ensure_npm, DevtoolError
Patrick Williamsc124f4f2015-09-15 14:41:29 -050034from devtool import parse_recipe
35
36logger = logging.getLogger('devtool')
37
38
39def add(args, config, basepath, workspace):
40 """Entry point for the devtool 'add' subcommand"""
41 import bb
42 import oe.recipeutils
43
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050044 if not args.recipename and not args.srctree and not args.fetch and not args.fetchuri:
45 raise argparse_oe.ArgumentUsageError('At least one of recipename, srctree, fetchuri or -f/--fetch must be specified', 'add')
Patrick Williamsc124f4f2015-09-15 14:41:29 -050046
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050047 # These are positional arguments, but because we're nice, allow
48 # specifying e.g. source tree without name, or fetch URI without name or
49 # source tree (if we can detect that that is what the user meant)
Patrick Williamsc0f7c042017-02-23 20:41:17 -060050 if scriptutils.is_src_url(args.recipename):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050051 if not args.fetchuri:
52 if args.fetch:
53 raise DevtoolError('URI specified as positional argument as well as -f/--fetch')
54 args.fetchuri = args.recipename
55 args.recipename = ''
Patrick Williamsc0f7c042017-02-23 20:41:17 -060056 elif scriptutils.is_src_url(args.srctree):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050057 if not args.fetchuri:
58 if args.fetch:
59 raise DevtoolError('URI specified as positional argument as well as -f/--fetch')
60 args.fetchuri = args.srctree
61 args.srctree = ''
62 elif args.recipename and not args.srctree:
63 if os.sep in args.recipename:
64 args.srctree = args.recipename
65 args.recipename = None
66 elif os.path.isdir(args.recipename):
Patrick Williamsc0f7c042017-02-23 20:41:17 -060067 logger.warn('Ambiguous argument "%s" - assuming you mean it to be the recipe name' % args.recipename)
68
69 if args.srctree and os.path.isfile(args.srctree):
70 args.fetchuri = 'file://' + os.path.abspath(args.srctree)
71 args.srctree = ''
Patrick Williamsc124f4f2015-09-15 14:41:29 -050072
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050073 if args.fetch:
74 if args.fetchuri:
75 raise DevtoolError('URI specified as positional argument as well as -f/--fetch')
76 else:
Patrick Williamsc0f7c042017-02-23 20:41:17 -060077 logger.warn('-f/--fetch option is deprecated - you can now simply specify the URL to fetch as a positional argument instead')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050078 args.fetchuri = args.fetch
Patrick Williamsf1e5d692016-03-30 15:21:19 -050079
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050080 if args.recipename:
81 if args.recipename in workspace:
82 raise DevtoolError("recipe %s is already in your workspace" %
83 args.recipename)
84 reason = oe.recipeutils.validate_pn(args.recipename)
85 if reason:
86 raise DevtoolError(reason)
87
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050088 if args.srctree:
89 srctree = os.path.abspath(args.srctree)
90 srctreeparent = None
91 tmpsrcdir = None
92 else:
93 srctree = None
94 srctreeparent = get_default_srctree(config)
95 bb.utils.mkdirhier(srctreeparent)
96 tmpsrcdir = tempfile.mkdtemp(prefix='devtoolsrc', dir=srctreeparent)
97
98 if srctree and os.path.exists(srctree):
99 if args.fetchuri:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500100 if not os.path.isdir(srctree):
101 raise DevtoolError("Cannot fetch into source tree path %s as "
102 "it exists and is not a directory" %
103 srctree)
104 elif os.listdir(srctree):
105 raise DevtoolError("Cannot fetch into source tree path %s as "
106 "it already exists and is non-empty" %
107 srctree)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500108 elif not args.fetchuri:
109 if args.srctree:
110 raise DevtoolError("Specified source tree %s could not be found" %
111 args.srctree)
112 elif srctree:
113 raise DevtoolError("No source tree exists at default path %s - "
114 "either create and populate this directory, "
115 "or specify a path to a source tree, or a "
116 "URI to fetch source from" % srctree)
117 else:
118 raise DevtoolError("You must either specify a source tree "
119 "or a URI to fetch source from")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500120
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500121 if args.version:
122 if '_' in args.version or ' ' in args.version:
123 raise DevtoolError('Invalid version string "%s"' % args.version)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500124
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500125 if args.color == 'auto' and sys.stdout.isatty():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500126 color = 'always'
127 else:
128 color = args.color
129 extracmdopts = ''
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500130 if args.fetchuri:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600131 if args.fetchuri.startswith('npm://'):
132 ensure_npm(config, basepath, args.fixed_setup)
133
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500134 source = args.fetchuri
135 if srctree:
136 extracmdopts += ' -x %s' % srctree
137 else:
138 extracmdopts += ' -x %s' % tmpsrcdir
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500139 else:
140 source = srctree
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500141 if args.recipename:
142 extracmdopts += ' -N %s' % args.recipename
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500143 if args.version:
144 extracmdopts += ' -V %s' % args.version
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500145 if args.binary:
146 extracmdopts += ' -b'
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500147 if args.also_native:
148 extracmdopts += ' --also-native'
149 if args.src_subdir:
150 extracmdopts += ' --src-subdir "%s"' % args.src_subdir
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600151 if args.autorev:
152 extracmdopts += ' -a'
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500153 if args.fetch_dev:
154 extracmdopts += ' --fetch-dev'
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500155
156 tempdir = tempfile.mkdtemp(prefix='devtool')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500157 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500158 builtnpm = False
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600159 while True:
160 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500161 stdout, _ = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create --devtool -o %s \'%s\' %s' % (color, tempdir, source, extracmdopts), watch=True)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600162 except bb.process.ExecutionError as e:
163 if e.exitcode == 14:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500164 if builtnpm:
165 raise DevtoolError('Re-running recipetool still failed to find npm')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600166 # FIXME this is a horrible hack that is unfortunately
167 # necessary due to the fact that we can't run bitbake from
168 # inside recipetool since recipetool keeps tinfoil active
169 # with references to it throughout the code, so we have
170 # to exit out and come back here to do it.
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500171 ensure_npm(config, basepath, args.fixed_setup, check_exists=False)
172 logger.info('Re-running recipe creation process after building nodejs')
173 builtnpm = True
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600174 continue
175 elif e.exitcode == 15:
176 raise DevtoolError('Could not auto-determine recipe name, please specify it on the command line')
177 else:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500178 raise DevtoolError('Command \'%s\' failed' % e.command)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600179 break
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500180
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500181 recipes = glob.glob(os.path.join(tempdir, '*.bb'))
182 if recipes:
183 recipename = os.path.splitext(os.path.basename(recipes[0]))[0].split('_')[0]
184 if recipename in workspace:
185 raise DevtoolError('A recipe with the same name as the one being created (%s) already exists in your workspace' % recipename)
186 recipedir = os.path.join(config.workspace_path, 'recipes', recipename)
187 bb.utils.mkdirhier(recipedir)
188 recipefile = os.path.join(recipedir, os.path.basename(recipes[0]))
189 appendfile = recipe_to_append(recipefile, config)
190 if os.path.exists(appendfile):
191 # This shouldn't be possible, but just in case
192 raise DevtoolError('A recipe with the same name as the one being created already exists in your workspace')
193 if os.path.exists(recipefile):
194 raise DevtoolError('A recipe file %s already exists in your workspace; this shouldn\'t be there - please delete it before continuing' % recipefile)
195 if tmpsrcdir:
196 srctree = os.path.join(srctreeparent, recipename)
197 if os.path.exists(tmpsrcdir):
198 if os.path.exists(srctree):
199 if os.path.isdir(srctree):
200 try:
201 os.rmdir(srctree)
202 except OSError as e:
203 if e.errno == errno.ENOTEMPTY:
204 raise DevtoolError('Source tree path %s already exists and is not empty' % srctree)
205 else:
206 raise
207 else:
208 raise DevtoolError('Source tree path %s already exists and is not a directory' % srctree)
209 logger.info('Using default source tree path %s' % srctree)
210 shutil.move(tmpsrcdir, srctree)
211 else:
212 raise DevtoolError('Couldn\'t find source tree created by recipetool')
213 bb.utils.mkdirhier(recipedir)
214 shutil.move(recipes[0], recipefile)
215 # Move any additional files created by recipetool
216 for fn in os.listdir(tempdir):
217 shutil.move(os.path.join(tempdir, fn), recipedir)
218 else:
219 raise DevtoolError('Command \'%s\' did not create any recipe file:\n%s' % (e.command, e.stdout))
220 attic_recipe = os.path.join(config.workspace_path, 'attic', recipename, os.path.basename(recipefile))
221 if os.path.exists(attic_recipe):
222 logger.warn('A modified recipe from a previous invocation exists in %s - you may wish to move this over the top of the new recipe if you had changes in it that you want to continue with' % attic_recipe)
223 finally:
224 if tmpsrcdir and os.path.exists(tmpsrcdir):
225 shutil.rmtree(tmpsrcdir)
226 shutil.rmtree(tempdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500227
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500228 for fn in os.listdir(recipedir):
229 _add_md5(config, recipename, os.path.join(recipedir, fn))
230
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500231 tinfoil = setup_tinfoil(config_only=True, basepath=basepath)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600232 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500233 try:
234 rd = tinfoil.parse_recipe_file(recipefile, False)
235 except Exception as e:
236 logger.error(str(e))
237 rd = None
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600238 if not rd:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500239 # Parsing failed. We just created this recipe and we shouldn't
240 # leave it in the workdir or it'll prevent bitbake from starting
241 movefn = '%s.parsefailed' % recipefile
242 logger.error('Parsing newly created recipe failed, moving recipe to %s for reference. If this looks to be caused by the recipe itself, please report this error.' % movefn)
243 shutil.move(recipefile, movefn)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600244 return 1
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500245
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600246 if args.fetchuri and not args.no_git:
247 setup_git_repo(srctree, args.version, 'devtool', d=tinfoil.config_data)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500248
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600249 initial_rev = None
250 if os.path.exists(os.path.join(srctree, '.git')):
251 (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree)
252 initial_rev = stdout.rstrip()
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500253
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600254 if args.src_subdir:
255 srctree = os.path.join(srctree, args.src_subdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500256
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600257 bb.utils.mkdirhier(os.path.dirname(appendfile))
258 with open(appendfile, 'w') as f:
259 f.write('inherit externalsrc\n')
260 f.write('EXTERNALSRC = "%s"\n' % srctree)
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500261
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600262 b_is_s = use_external_build(args.same_dir, args.no_same_dir, rd)
263 if b_is_s:
264 f.write('EXTERNALSRC_BUILD = "%s"\n' % srctree)
265 if initial_rev:
266 f.write('\n# initial_rev: %s\n' % initial_rev)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500267
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600268 if args.binary:
269 f.write('do_install_append() {\n')
270 f.write(' rm -rf ${D}/.git\n')
271 f.write(' rm -f ${D}/singletask.lock\n')
272 f.write('}\n')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500273
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600274 if bb.data.inherits_class('npm', rd):
275 f.write('do_install_append() {\n')
276 f.write(' # Remove files added to source dir by devtool/externalsrc\n')
277 f.write(' rm -f ${NPM_INSTALLDIR}/singletask.lock\n')
278 f.write(' rm -rf ${NPM_INSTALLDIR}/.git\n')
279 f.write(' rm -rf ${NPM_INSTALLDIR}/oe-local-files\n')
280 f.write(' for symlink in ${EXTERNALSRC_SYMLINKS} ; do\n')
281 f.write(' rm -f ${NPM_INSTALLDIR}/${symlink%%:*}\n')
282 f.write(' done\n')
283 f.write('}\n')
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500284
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600285 _add_md5(config, recipename, appendfile)
286
287 logger.info('Recipe %s has been automatically created; further editing may be required to make it fully functional' % recipefile)
288
289 finally:
290 tinfoil.shutdown()
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500291
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500292 return 0
293
294
295def _check_compatible_recipe(pn, d):
296 """Check if the recipe is supported by devtool"""
297 if pn == 'perf':
298 raise DevtoolError("The perf recipe does not actually check out "
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600299 "source and thus cannot be supported by this tool",
300 4)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500301
302 if pn in ['kernel-devsrc', 'package-index'] or pn.startswith('gcc-source'):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600303 raise DevtoolError("The %s recipe is not supported by this tool" % pn, 4)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500304
305 if bb.data.inherits_class('image', d):
306 raise DevtoolError("The %s recipe is an image, and therefore is not "
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600307 "supported by this tool" % pn, 4)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500308
309 if bb.data.inherits_class('populate_sdk', d):
310 raise DevtoolError("The %s recipe is an SDK, and therefore is not "
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600311 "supported by this tool" % pn, 4)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500312
313 if bb.data.inherits_class('packagegroup', d):
314 raise DevtoolError("The %s recipe is a packagegroup, and therefore is "
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600315 "not supported by this tool" % pn, 4)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500316
317 if bb.data.inherits_class('meta', d):
318 raise DevtoolError("The %s recipe is a meta-recipe, and therefore is "
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600319 "not supported by this tool" % pn, 4)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500320
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500321 if bb.data.inherits_class('externalsrc', d) and d.getVar('EXTERNALSRC'):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600322 # Not an incompatibility error per se, so we don't pass the error code
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500323 raise DevtoolError("externalsrc is currently enabled for the %s "
324 "recipe. This prevents the normal do_patch task "
325 "from working. You will need to disable this "
326 "first." % pn)
327
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500328def _move_file(src, dst):
329 """Move a file. Creates all the directory components of destination path."""
330 dst_d = os.path.dirname(dst)
331 if dst_d:
332 bb.utils.mkdirhier(dst_d)
333 shutil.move(src, dst)
334
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600335def _copy_file(src, dst):
336 """Copy a file. Creates all the directory components of destination path."""
337 dst_d = os.path.dirname(dst)
338 if dst_d:
339 bb.utils.mkdirhier(dst_d)
340 shutil.copy(src, dst)
341
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500342def _git_ls_tree(repodir, treeish='HEAD', recursive=False):
343 """List contents of a git treeish"""
344 import bb
345 cmd = ['git', 'ls-tree', '-z', treeish]
346 if recursive:
347 cmd.append('-r')
348 out, _ = bb.process.run(cmd, cwd=repodir)
349 ret = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500350 if out:
351 for line in out.split('\0'):
352 if line:
353 split = line.split(None, 4)
354 ret[split[3]] = split[0:3]
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500355 return ret
356
357def _git_exclude_path(srctree, path):
358 """Return pathspec (list of paths) that excludes certain path"""
359 # NOTE: "Filtering out" files/paths in this way is not entirely reliable -
360 # we don't catch files that are deleted, for example. A more reliable way
361 # to implement this would be to use "negative pathspecs" which were
362 # introduced in Git v1.9.0. Revisit this when/if the required Git version
363 # becomes greater than that.
364 path = os.path.normpath(path)
365 recurse = True if len(path.split(os.path.sep)) > 1 else False
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600366 git_files = list(_git_ls_tree(srctree, 'HEAD', recurse).keys())
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500367 if path in git_files:
368 git_files.remove(path)
369 return git_files
370 else:
371 return ['.']
372
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500373def _ls_tree(directory):
374 """Recursive listing of files in a directory"""
375 ret = []
376 for root, dirs, files in os.walk(directory):
377 ret.extend([os.path.relpath(os.path.join(root, fname), directory) for
378 fname in files])
379 return ret
380
381
382def extract(args, config, basepath, workspace):
383 """Entry point for the devtool 'extract' subcommand"""
384 import bb
385
386 tinfoil = _prep_extract_operation(config, basepath, args.recipename)
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500387 if not tinfoil:
388 # Error already shown
389 return 1
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600390 try:
391 rd = parse_recipe(config, tinfoil, args.recipename, True)
392 if not rd:
393 return 1
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500394
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600395 srctree = os.path.abspath(args.srctree)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500396 initial_rev = _extract_source(srctree, args.keep_temp, args.branch, False, rd, tinfoil)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600397 logger.info('Source tree extracted to %s' % srctree)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500398
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600399 if initial_rev:
400 return 0
401 else:
402 return 1
403 finally:
404 tinfoil.shutdown()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500405
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500406def sync(args, config, basepath, workspace):
407 """Entry point for the devtool 'sync' subcommand"""
408 import bb
409
410 tinfoil = _prep_extract_operation(config, basepath, args.recipename)
411 if not tinfoil:
412 # Error already shown
413 return 1
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600414 try:
415 rd = parse_recipe(config, tinfoil, args.recipename, True)
416 if not rd:
417 return 1
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500418
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600419 srctree = os.path.abspath(args.srctree)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500420 initial_rev = _extract_source(srctree, args.keep_temp, args.branch, True, rd, tinfoil)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600421 logger.info('Source tree %s synchronized' % srctree)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500422
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600423 if initial_rev:
424 return 0
425 else:
426 return 1
427 finally:
428 tinfoil.shutdown()
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500429
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500430
431def _prep_extract_operation(config, basepath, recipename, tinfoil=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500432 """HACK: Ugly workaround for making sure that requirements are met when
433 trying to extract a package. Returns the tinfoil instance to be used."""
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500434 if not tinfoil:
435 tinfoil = setup_tinfoil(basepath=basepath)
436
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500437 rd = parse_recipe(config, tinfoil, recipename, True)
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500438 if not rd:
439 return None
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500440
441 if bb.data.inherits_class('kernel-yocto', rd):
442 tinfoil.shutdown()
443 try:
444 stdout, _ = exec_build_env_command(config.init_path, basepath,
445 'bitbake kern-tools-native')
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500446 tinfoil = setup_tinfoil(basepath=basepath)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500447 except bb.process.ExecutionError as err:
448 raise DevtoolError("Failed to build kern-tools-native:\n%s" %
449 err.stdout)
450 return tinfoil
451
452
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500453def _extract_source(srctree, keep_temp, devbranch, sync, d, tinfoil):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500454 """Extract sources of a recipe"""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500455 import oe.recipeutils
456
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500457 pn = d.getVar('PN')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500458
459 _check_compatible_recipe(pn, d)
460
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500461 if sync:
462 if not os.path.exists(srctree):
463 raise DevtoolError("output path %s does not exist" % srctree)
464 else:
465 if os.path.exists(srctree):
466 if not os.path.isdir(srctree):
467 raise DevtoolError("output path %s exists and is not a directory" %
468 srctree)
469 elif os.listdir(srctree):
470 raise DevtoolError("output path %s already exists and is "
471 "non-empty" % srctree)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500472
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500473 if 'noexec' in (d.getVarFlags('do_unpack', False) or []):
474 raise DevtoolError("The %s recipe has do_unpack disabled, unable to "
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600475 "extract source" % pn, 4)
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500476
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500477 if not sync:
478 # Prepare for shutil.move later on
479 bb.utils.mkdirhier(srctree)
480 os.rmdir(srctree)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500481
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500482 initial_rev = None
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500483 # We need to redirect WORKDIR, STAMPS_DIR etc. under a temporary
484 # directory so that:
485 # (a) we pick up all files that get unpacked to the WORKDIR, and
486 # (b) we don't disturb the existing build
487 # However, with recipe-specific sysroots the sysroots for the recipe
488 # will be prepared under WORKDIR, and if we used the system temporary
489 # directory (i.e. usually /tmp) as used by mkdtemp by default, then
490 # our attempts to hardlink files into the recipe-specific sysroots
491 # will fail on systems where /tmp is a different filesystem, and it
492 # would have to fall back to copying the files which is a waste of
493 # time. Put the temp directory under the WORKDIR to prevent that from
494 # being a problem.
495 tempbasedir = d.getVar('WORKDIR')
496 bb.utils.mkdirhier(tempbasedir)
497 tempdir = tempfile.mkdtemp(prefix='devtooltmp-', dir=tempbasedir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500498 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500499 tinfoil.logger.setLevel(logging.WARNING)
500
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500501 crd = d.createCopy()
502 # Make a subdir so we guard against WORKDIR==S
503 workdir = os.path.join(tempdir, 'workdir')
504 crd.setVar('WORKDIR', workdir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500505 if not crd.getVar('S').startswith(workdir):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500506 # Usually a shared workdir recipe (kernel, gcc)
507 # Try to set a reasonable default
508 if bb.data.inherits_class('kernel', d):
509 crd.setVar('S', '${WORKDIR}/source')
510 else:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500511 crd.setVar('S', '${WORKDIR}/%s' % os.path.basename(d.getVar('S')))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500512 if bb.data.inherits_class('kernel', d):
513 # We don't want to move the source to STAGING_KERNEL_DIR here
514 crd.setVar('STAGING_KERNEL_DIR', '${S}')
515
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500516 is_kernel_yocto = bb.data.inherits_class('kernel-yocto', d)
517 if not is_kernel_yocto:
518 crd.setVar('PATCHTOOL', 'git')
519 crd.setVar('PATCH_COMMIT_FUNCTIONS', '1')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500520
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500521 # Apply our changes to the datastore to the server's datastore
522 for key in crd.localkeys():
523 tinfoil.config_data.setVar('%s_pn-%s' % (key, pn), crd.getVar(key, False))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500524
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500525 tinfoil.config_data.setVar('STAMPS_DIR', os.path.join(tempdir, 'stamps'))
526 tinfoil.config_data.setVar('T', os.path.join(tempdir, 'temp'))
527 tinfoil.config_data.setVar('BUILDCFG_FUNCS', '')
528 tinfoil.config_data.setVar('BUILDCFG_HEADER', '')
529 tinfoil.config_data.setVar('BB_HASH_IGNORE_MISMATCH', '1')
530
531 tinfoil.set_event_mask(['bb.event.BuildStarted',
532 'bb.event.BuildCompleted',
533 'logging.LogRecord',
534 'bb.command.CommandCompleted',
535 'bb.command.CommandFailed',
536 'bb.build.TaskStarted',
537 'bb.build.TaskSucceeded',
538 'bb.build.TaskFailed',
539 'bb.build.TaskFailedSilent'])
540
541 def runtask(target, task):
542 if tinfoil.build_file(target, task):
543 while True:
544 event = tinfoil.wait_event(0.25)
545 if event:
546 if isinstance(event, bb.command.CommandCompleted):
547 break
548 elif isinstance(event, bb.command.CommandFailed):
549 raise DevtoolError('Task do_%s failed: %s' % (task, event.error))
550 elif isinstance(event, bb.build.TaskFailed):
551 raise DevtoolError('Task do_%s failed' % task)
552 elif isinstance(event, bb.build.TaskStarted):
553 logger.info('Executing %s...' % event._task)
554 elif isinstance(event, logging.LogRecord):
555 if event.levelno <= logging.INFO:
556 continue
557 logger.handle(event)
558
559 # we need virtual:native:/path/to/recipe if it's a BBCLASSEXTEND
560 fn = tinfoil.get_recipe_file(pn)
561 runtask(fn, 'unpack')
562
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500563 if bb.data.inherits_class('kernel-yocto', d):
564 # Extra step for kernel to populate the source directory
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500565 runtask(fn, 'kernel_checkout')
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500566
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500567 srcsubdir = crd.getVar('S')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500568
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500569 # Move local source files into separate subdir
570 recipe_patches = [os.path.basename(patch) for patch in
571 oe.recipeutils.get_recipe_patches(crd)]
572 local_files = oe.recipeutils.get_recipe_local_files(crd)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600573
574 # Ignore local files with subdir={BP}
575 srcabspath = os.path.abspath(srcsubdir)
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500576 local_files = [fname for fname in local_files if
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600577 os.path.exists(os.path.join(workdir, fname)) and
578 (srcabspath == workdir or not
579 os.path.join(workdir, fname).startswith(srcabspath +
580 os.sep))]
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500581 if local_files:
582 for fname in local_files:
583 _move_file(os.path.join(workdir, fname),
584 os.path.join(tempdir, 'oe-local-files', fname))
585 with open(os.path.join(tempdir, 'oe-local-files', '.gitignore'),
586 'w') as f:
587 f.write('# Ignore local files, by default. Remove this file '
588 'if you want to commit the directory to Git\n*\n')
589
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500590 if srcsubdir == workdir:
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500591 # Find non-patch non-local sources that were "unpacked" to srctree
592 # directory
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500593 src_files = [fname for fname in _ls_tree(workdir) if
594 os.path.basename(fname) not in recipe_patches]
595 # Force separate S so that patch files can be left out from srctree
596 srcsubdir = tempfile.mkdtemp(dir=workdir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500597 tinfoil.config_data.setVar('S_task-patch', srcsubdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500598 # Move source files to S
599 for path in src_files:
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500600 _move_file(os.path.join(workdir, path),
601 os.path.join(srcsubdir, path))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500602 elif os.path.dirname(srcsubdir) != workdir:
603 # Handle if S is set to a subdirectory of the source
604 srcsubdir = os.path.join(workdir, os.path.relpath(srcsubdir, workdir).split(os.sep)[0])
605
606 scriptutils.git_convert_standalone_clone(srcsubdir)
607
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500608 # Make sure that srcsubdir exists
609 bb.utils.mkdirhier(srcsubdir)
610 if not os.path.exists(srcsubdir) or not os.listdir(srcsubdir):
611 logger.warning("no source unpacked to S, either the %s recipe "
612 "doesn't use any source or the correct source "
613 "directory could not be determined" % pn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500614
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500615 setup_git_repo(srcsubdir, crd.getVar('PV'), devbranch, d=d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500616
617 (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srcsubdir)
618 initial_rev = stdout.rstrip()
619
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500620 logger.info('Patching...')
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500621 runtask(fn, 'patch')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500622
623 bb.process.run('git tag -f devtool-patched', cwd=srcsubdir)
624
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500625 kconfig = None
626 if bb.data.inherits_class('kernel-yocto', d):
627 # Store generate and store kernel config
628 logger.info('Generating kernel config')
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500629 runtask(fn, 'configure')
630 kconfig = os.path.join(crd.getVar('B'), '.config')
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500631
632
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500633 tempdir_localdir = os.path.join(tempdir, 'oe-local-files')
634 srctree_localdir = os.path.join(srctree, 'oe-local-files')
635
636 if sync:
637 bb.process.run('git fetch file://' + srcsubdir + ' ' + devbranch + ':' + devbranch, cwd=srctree)
638
639 # Move oe-local-files directory to srctree
640 # As the oe-local-files is not part of the constructed git tree,
641 # remove them directly during the synchrounizating might surprise
642 # the users. Instead, we move it to oe-local-files.bak and remind
643 # user in the log message.
644 if os.path.exists(srctree_localdir + '.bak'):
645 shutil.rmtree(srctree_localdir, srctree_localdir + '.bak')
646
647 if os.path.exists(srctree_localdir):
648 logger.info('Backing up current local file directory %s' % srctree_localdir)
649 shutil.move(srctree_localdir, srctree_localdir + '.bak')
650
651 if os.path.exists(tempdir_localdir):
652 logger.info('Syncing local source files to srctree...')
653 shutil.copytree(tempdir_localdir, srctree_localdir)
654 else:
655 # Move oe-local-files directory to srctree
656 if os.path.exists(tempdir_localdir):
657 logger.info('Adding local source files to srctree...')
658 shutil.move(tempdir_localdir, srcsubdir)
659
660 shutil.move(srcsubdir, srctree)
661
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500662 if os.path.abspath(d.getVar('S')) == os.path.abspath(d.getVar('WORKDIR')):
663 # If recipe extracts to ${WORKDIR}, symlink the files into the srctree
664 # (otherwise the recipe won't build as expected)
665 local_files_dir = os.path.join(srctree, 'oe-local-files')
666 addfiles = []
667 for root, _, files in os.walk(local_files_dir):
668 relpth = os.path.relpath(root, local_files_dir)
669 if relpth != '.':
670 bb.utils.mkdirhier(os.path.join(srctree, relpth))
671 for fn in files:
672 if fn == '.gitignore':
673 continue
674 destpth = os.path.join(srctree, relpth, fn)
675 if os.path.exists(destpth):
676 os.unlink(destpth)
677 os.symlink('oe-local-files/%s' % fn, destpth)
678 addfiles.append(os.path.join(relpth, fn))
679 if addfiles:
680 bb.process.run('git add %s' % ' '.join(addfiles), cwd=srctree)
681 useroptions = []
682 oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=d)
683 bb.process.run('git %s commit -a -m "Committing local file symlinks\n\n%s"' % (' '.join(useroptions), oe.patch.GitApplyTree.ignore_commit_prefix), cwd=srctree)
684
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500685 if kconfig:
686 logger.info('Copying kernel config to srctree')
687 shutil.copy2(kconfig, srctree)
688
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500689 finally:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500690 if keep_temp:
691 logger.info('Preserving temporary directory %s' % tempdir)
692 else:
693 shutil.rmtree(tempdir)
694 return initial_rev
695
696def _add_md5(config, recipename, filename):
697 """Record checksum of a file (or recursively for a directory) to the md5-file of the workspace"""
698 import bb.utils
699
700 def addfile(fn):
701 md5 = bb.utils.md5_file(fn)
702 with open(os.path.join(config.workspace_path, '.devtool_md5'), 'a') as f:
703 f.write('%s|%s|%s\n' % (recipename, os.path.relpath(fn, config.workspace_path), md5))
704
705 if os.path.isdir(filename):
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500706 for root, _, files in os.walk(filename):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500707 for f in files:
708 addfile(os.path.join(root, f))
709 else:
710 addfile(filename)
711
712def _check_preserve(config, recipename):
713 """Check if a file was manually changed and needs to be saved in 'attic'
714 directory"""
715 import bb.utils
716 origfile = os.path.join(config.workspace_path, '.devtool_md5')
717 newfile = os.path.join(config.workspace_path, '.devtool_md5_new')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500718 preservepath = os.path.join(config.workspace_path, 'attic', recipename)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500719 with open(origfile, 'r') as f:
720 with open(newfile, 'w') as tf:
721 for line in f.readlines():
722 splitline = line.rstrip().split('|')
723 if splitline[0] == recipename:
724 removefile = os.path.join(config.workspace_path, splitline[1])
725 try:
726 md5 = bb.utils.md5_file(removefile)
727 except IOError as err:
728 if err.errno == 2:
729 # File no longer exists, skip it
730 continue
731 else:
732 raise
733 if splitline[2] != md5:
734 bb.utils.mkdirhier(preservepath)
735 preservefile = os.path.basename(removefile)
736 logger.warn('File %s modified since it was written, preserving in %s' % (preservefile, preservepath))
737 shutil.move(removefile, os.path.join(preservepath, preservefile))
738 else:
739 os.remove(removefile)
740 else:
741 tf.write(line)
742 os.rename(newfile, origfile)
743
744def modify(args, config, basepath, workspace):
745 """Entry point for the devtool 'modify' subcommand"""
746 import bb
747 import oe.recipeutils
748
749 if args.recipename in workspace:
750 raise DevtoolError("recipe %s is already in your workspace" %
751 args.recipename)
752
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500753 tinfoil = setup_tinfoil(basepath=basepath)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600754 try:
755 rd = parse_recipe(config, tinfoil, args.recipename, True)
756 if not rd:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500757 return 1
758
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500759 pn = rd.getVar('PN')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600760 if pn != args.recipename:
761 logger.info('Mapping %s to %s' % (args.recipename, pn))
762 if pn in workspace:
763 raise DevtoolError("recipe %s is already in your workspace" %
764 pn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500765
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600766 if args.srctree:
767 srctree = os.path.abspath(args.srctree)
768 else:
769 srctree = get_default_srctree(config, pn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500770
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600771 if args.no_extract and not os.path.isdir(srctree):
772 raise DevtoolError("--no-extract specified and source path %s does "
773 "not exist or is not a directory" %
774 srctree)
775 if not args.no_extract:
776 tinfoil = _prep_extract_operation(config, basepath, pn, tinfoil)
777 if not tinfoil:
778 # Error already shown
779 return 1
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500780 # We need to re-parse because tinfoil may have been re-initialised
781 rd = parse_recipe(config, tinfoil, args.recipename, True)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600782
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500783 recipefile = rd.getVar('FILE')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600784 appendfile = recipe_to_append(recipefile, config, args.wildcard)
785 if os.path.exists(appendfile):
786 raise DevtoolError("Another variant of recipe %s is already in your "
787 "workspace (only one variant of a recipe can "
788 "currently be worked on at once)"
789 % pn)
790
791 _check_compatible_recipe(pn, rd)
792
793 initial_rev = None
794 commits = []
795 if not args.no_extract:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500796 initial_rev = _extract_source(srctree, args.keep_temp, args.branch, False, rd, tinfoil)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500797 if not initial_rev:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600798 return 1
799 logger.info('Source tree extracted to %s' % srctree)
800 # Get list of commits since this revision
801 (stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % initial_rev, cwd=srctree)
802 commits = stdout.split()
803 else:
804 if os.path.exists(os.path.join(srctree, '.git')):
805 # Check if it's a tree previously extracted by us
806 try:
807 (stdout, _) = bb.process.run('git branch --contains devtool-base', cwd=srctree)
808 except bb.process.ExecutionError:
809 stdout = ''
810 for line in stdout.splitlines():
811 if line.startswith('*'):
812 (stdout, _) = bb.process.run('git rev-parse devtool-base', cwd=srctree)
813 initial_rev = stdout.rstrip()
814 if not initial_rev:
815 # Otherwise, just grab the head revision
816 (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree)
817 initial_rev = stdout.rstrip()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500818
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600819 # Check that recipe isn't using a shared workdir
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500820 s = os.path.abspath(rd.getVar('S'))
821 workdir = os.path.abspath(rd.getVar('WORKDIR'))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600822 if s.startswith(workdir) and s != workdir and os.path.dirname(s) != workdir:
823 # Handle if S is set to a subdirectory of the source
824 srcsubdir = os.path.relpath(s, workdir).split(os.sep, 1)[1]
825 srctree = os.path.join(srctree, srcsubdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500826
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600827 bb.utils.mkdirhier(os.path.dirname(appendfile))
828 with open(appendfile, 'w') as f:
829 f.write('FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n')
830 # Local files can be modified/tracked in separate subdir under srctree
831 # Mostly useful for packages with S != WORKDIR
832 f.write('FILESPATH_prepend := "%s:"\n' %
833 os.path.join(srctree, 'oe-local-files'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500834
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600835 f.write('\ninherit externalsrc\n')
836 f.write('# NOTE: We use pn- overrides here to avoid affecting multiple variants in the case where the recipe uses BBCLASSEXTEND\n')
837 f.write('EXTERNALSRC_pn-%s = "%s"\n' % (pn, srctree))
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500838
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600839 b_is_s = use_external_build(args.same_dir, args.no_same_dir, rd)
840 if b_is_s:
841 f.write('EXTERNALSRC_BUILD_pn-%s = "%s"\n' % (pn, srctree))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500842
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600843 if bb.data.inherits_class('kernel', rd):
844 f.write('SRCTREECOVEREDTASKS = "do_validate_branches do_kernel_checkout '
845 'do_fetch do_unpack do_patch do_kernel_configme do_kernel_configcheck"\n')
846 f.write('\ndo_configure_append() {\n'
847 ' cp ${B}/.config ${S}/.config.baseline\n'
848 ' ln -sfT ${B}/.config ${S}/.config.new\n'
849 '}\n')
850 if initial_rev:
851 f.write('\n# initial_rev: %s\n' % initial_rev)
852 for commit in commits:
853 f.write('# commit: %s\n' % commit)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500854
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600855 _add_md5(config, pn, appendfile)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500856
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600857 logger.info('Recipe %s now set up to build from %s' % (pn, srctree))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500858
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600859 finally:
860 tinfoil.shutdown()
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500861
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500862 return 0
863
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500864
865def rename(args, config, basepath, workspace):
866 """Entry point for the devtool 'rename' subcommand"""
867 import bb
868 import oe.recipeutils
869
870 check_workspace_recipe(workspace, args.recipename)
871
872 if not (args.newname or args.version):
873 raise DevtoolError('You must specify a new name, a version with -V/--version, or both')
874
875 recipefile = workspace[args.recipename]['recipefile']
876 if not recipefile:
877 raise DevtoolError('devtool rename can only be used where the recipe file itself is in the workspace (e.g. after devtool add)')
878
879 if args.newname and args.newname != args.recipename:
880 reason = oe.recipeutils.validate_pn(args.newname)
881 if reason:
882 raise DevtoolError(reason)
883 newname = args.newname
884 else:
885 newname = args.recipename
886
887 append = workspace[args.recipename]['bbappend']
888 appendfn = os.path.splitext(os.path.basename(append))[0]
889 splitfn = appendfn.split('_')
890 if len(splitfn) > 1:
891 origfnver = appendfn.split('_')[1]
892 else:
893 origfnver = ''
894
895 recipefilemd5 = None
896 tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
897 try:
898 rd = parse_recipe(config, tinfoil, args.recipename, True)
899 if not rd:
900 return 1
901
902 bp = rd.getVar('BP')
903 bpn = rd.getVar('BPN')
904 if newname != args.recipename:
905 localdata = rd.createCopy()
906 localdata.setVar('PN', newname)
907 newbpn = localdata.getVar('BPN')
908 else:
909 newbpn = bpn
910 s = rd.getVar('S', False)
911 src_uri = rd.getVar('SRC_URI', False)
912 pv = rd.getVar('PV')
913
914 # Correct variable values that refer to the upstream source - these
915 # values must stay the same, so if the name/version are changing then
916 # we need to fix them up
917 new_s = s
918 new_src_uri = src_uri
919 if newbpn != bpn:
920 # ${PN} here is technically almost always incorrect, but people do use it
921 new_s = new_s.replace('${BPN}', bpn)
922 new_s = new_s.replace('${PN}', bpn)
923 new_s = new_s.replace('${BP}', '%s-${PV}' % bpn)
924 new_src_uri = new_src_uri.replace('${BPN}', bpn)
925 new_src_uri = new_src_uri.replace('${PN}', bpn)
926 new_src_uri = new_src_uri.replace('${BP}', '%s-${PV}' % bpn)
927 if args.version and origfnver == pv:
928 new_s = new_s.replace('${PV}', pv)
929 new_s = new_s.replace('${BP}', '${BPN}-%s' % pv)
930 new_src_uri = new_src_uri.replace('${PV}', pv)
931 new_src_uri = new_src_uri.replace('${BP}', '${BPN}-%s' % pv)
932 patchfields = {}
933 if new_s != s:
934 patchfields['S'] = new_s
935 if new_src_uri != src_uri:
936 patchfields['SRC_URI'] = new_src_uri
937 if patchfields:
938 recipefilemd5 = bb.utils.md5_file(recipefile)
939 oe.recipeutils.patch_recipe(rd, recipefile, patchfields)
940 newrecipefilemd5 = bb.utils.md5_file(recipefile)
941 finally:
942 tinfoil.shutdown()
943
944 if args.version:
945 newver = args.version
946 else:
947 newver = origfnver
948
949 if newver:
950 newappend = '%s_%s.bbappend' % (newname, newver)
951 newfile = '%s_%s.bb' % (newname, newver)
952 else:
953 newappend = '%s.bbappend' % newname
954 newfile = '%s.bb' % newname
955
956 oldrecipedir = os.path.dirname(recipefile)
957 newrecipedir = os.path.join(config.workspace_path, 'recipes', newname)
958 if oldrecipedir != newrecipedir:
959 bb.utils.mkdirhier(newrecipedir)
960
961 newappend = os.path.join(os.path.dirname(append), newappend)
962 newfile = os.path.join(newrecipedir, newfile)
963
964 # Rename bbappend
965 logger.info('Renaming %s to %s' % (append, newappend))
966 os.rename(append, newappend)
967 # Rename recipe file
968 logger.info('Renaming %s to %s' % (recipefile, newfile))
969 os.rename(recipefile, newfile)
970
971 # Rename source tree if it's the default path
972 appendmd5 = None
973 if not args.no_srctree:
974 srctree = workspace[args.recipename]['srctree']
975 if os.path.abspath(srctree) == os.path.join(config.workspace_path, 'sources', args.recipename):
976 newsrctree = os.path.join(config.workspace_path, 'sources', newname)
977 logger.info('Renaming %s to %s' % (srctree, newsrctree))
978 shutil.move(srctree, newsrctree)
979 # Correct any references (basically EXTERNALSRC*) in the .bbappend
980 appendmd5 = bb.utils.md5_file(newappend)
981 appendlines = []
982 with open(newappend, 'r') as f:
983 for line in f:
984 appendlines.append(line)
985 with open(newappend, 'w') as f:
986 for line in appendlines:
987 if srctree in line:
988 line = line.replace(srctree, newsrctree)
989 f.write(line)
990 newappendmd5 = bb.utils.md5_file(newappend)
991
992 bpndir = None
993 newbpndir = None
994 if newbpn != bpn:
995 bpndir = os.path.join(oldrecipedir, bpn)
996 if os.path.exists(bpndir):
997 newbpndir = os.path.join(newrecipedir, newbpn)
998 logger.info('Renaming %s to %s' % (bpndir, newbpndir))
999 shutil.move(bpndir, newbpndir)
1000
1001 bpdir = None
1002 newbpdir = None
1003 if newver != origfnver or newbpn != bpn:
1004 bpdir = os.path.join(oldrecipedir, bp)
1005 if os.path.exists(bpdir):
1006 newbpdir = os.path.join(newrecipedir, '%s-%s' % (newbpn, newver))
1007 logger.info('Renaming %s to %s' % (bpdir, newbpdir))
1008 shutil.move(bpdir, newbpdir)
1009
1010 if oldrecipedir != newrecipedir:
1011 # Move any stray files and delete the old recipe directory
1012 for entry in os.listdir(oldrecipedir):
1013 oldpath = os.path.join(oldrecipedir, entry)
1014 newpath = os.path.join(newrecipedir, entry)
1015 logger.info('Renaming %s to %s' % (oldpath, newpath))
1016 shutil.move(oldpath, newpath)
1017 os.rmdir(oldrecipedir)
1018
1019 # Now take care of entries in .devtool_md5
1020 md5entries = []
1021 with open(os.path.join(config.workspace_path, '.devtool_md5'), 'r') as f:
1022 for line in f:
1023 md5entries.append(line)
1024
1025 if bpndir and newbpndir:
1026 relbpndir = os.path.relpath(bpndir, config.workspace_path) + '/'
1027 else:
1028 relbpndir = None
1029 if bpdir and newbpdir:
1030 relbpdir = os.path.relpath(bpdir, config.workspace_path) + '/'
1031 else:
1032 relbpdir = None
1033
1034 with open(os.path.join(config.workspace_path, '.devtool_md5'), 'w') as f:
1035 for entry in md5entries:
1036 splitentry = entry.rstrip().split('|')
1037 if len(splitentry) > 2:
1038 if splitentry[0] == args.recipename:
1039 splitentry[0] = newname
1040 if splitentry[1] == os.path.relpath(append, config.workspace_path):
1041 splitentry[1] = os.path.relpath(newappend, config.workspace_path)
1042 if appendmd5 and splitentry[2] == appendmd5:
1043 splitentry[2] = newappendmd5
1044 elif splitentry[1] == os.path.relpath(recipefile, config.workspace_path):
1045 splitentry[1] = os.path.relpath(newfile, config.workspace_path)
1046 if recipefilemd5 and splitentry[2] == recipefilemd5:
1047 splitentry[2] = newrecipefilemd5
1048 elif relbpndir and splitentry[1].startswith(relbpndir):
1049 splitentry[1] = os.path.relpath(os.path.join(newbpndir, splitentry[1][len(relbpndir):]), config.workspace_path)
1050 elif relbpdir and splitentry[1].startswith(relbpdir):
1051 splitentry[1] = os.path.relpath(os.path.join(newbpdir, splitentry[1][len(relbpdir):]), config.workspace_path)
1052 entry = '|'.join(splitentry) + '\n'
1053 f.write(entry)
1054 return 0
1055
1056
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001057def _get_patchset_revs(srctree, recipe_path, initial_rev=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001058 """Get initial and update rev of a recipe. These are the start point of the
1059 whole patchset and start point for the patches to be re-generated/updated.
1060 """
1061 import bb
1062
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001063 # Parse initial rev from recipe if not specified
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001064 commits = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001065 with open(recipe_path, 'r') as f:
1066 for line in f:
1067 if line.startswith('# initial_rev:'):
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001068 if not initial_rev:
1069 initial_rev = line.split(':')[-1].strip()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001070 elif line.startswith('# commit:'):
1071 commits.append(line.split(':')[-1].strip())
1072
1073 update_rev = initial_rev
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001074 changed_revs = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001075 if initial_rev:
1076 # Find first actually changed revision
1077 stdout, _ = bb.process.run('git rev-list --reverse %s..HEAD' %
1078 initial_rev, cwd=srctree)
1079 newcommits = stdout.split()
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001080 for i in range(min(len(commits), len(newcommits))):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001081 if newcommits[i] == commits[i]:
1082 update_rev = commits[i]
1083
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001084 try:
1085 stdout, _ = bb.process.run('git cherry devtool-patched',
1086 cwd=srctree)
1087 except bb.process.ExecutionError as err:
1088 stdout = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001089
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001090 if stdout is not None:
1091 changed_revs = []
1092 for line in stdout.splitlines():
1093 if line.startswith('+ '):
1094 rev = line.split()[1]
1095 if rev in newcommits:
1096 changed_revs.append(rev)
1097
1098 return initial_rev, update_rev, changed_revs
1099
1100def _remove_file_entries(srcuri, filelist):
1101 """Remove file:// entries from SRC_URI"""
1102 remaining = filelist[:]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001103 entries = []
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001104 for fname in filelist:
1105 basename = os.path.basename(fname)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001106 for i in range(len(srcuri)):
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001107 if (srcuri[i].startswith('file://') and
1108 os.path.basename(srcuri[i].split(';')[0]) == basename):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001109 entries.append(srcuri[i])
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001110 remaining.remove(fname)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001111 srcuri.pop(i)
1112 break
1113 return entries, remaining
1114
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001115def _replace_srcuri_entry(srcuri, filename, newentry):
1116 """Replace entry corresponding to specified file with a new entry"""
1117 basename = os.path.basename(filename)
1118 for i in range(len(srcuri)):
1119 if os.path.basename(srcuri[i].split(';')[0]) == basename:
1120 srcuri.pop(i)
1121 srcuri.insert(i, newentry)
1122 break
1123
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001124def _remove_source_files(append, files, destpath):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001125 """Unlink existing patch files"""
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001126 for path in files:
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001127 if append:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001128 if not destpath:
1129 raise Exception('destpath should be set here')
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001130 path = os.path.join(destpath, os.path.basename(path))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001131
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001132 if os.path.exists(path):
1133 logger.info('Removing file %s' % path)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001134 # FIXME "git rm" here would be nice if the file in question is
1135 # tracked
1136 # FIXME there's a chance that this file is referred to by
1137 # another recipe, in which case deleting wouldn't be the
1138 # right thing to do
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001139 os.remove(path)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001140 # Remove directory if empty
1141 try:
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001142 os.rmdir(os.path.dirname(path))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001143 except OSError as ose:
1144 if ose.errno != errno.ENOTEMPTY:
1145 raise
1146
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001147
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001148def _export_patches(srctree, rd, start_rev, destdir, changed_revs=None):
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001149 """Export patches from srctree to given location.
1150 Returns three-tuple of dicts:
1151 1. updated - patches that already exist in SRCURI
1152 2. added - new patches that don't exist in SRCURI
1153 3 removed - patches that exist in SRCURI but not in exported patches
1154 In each dict the key is the 'basepath' of the URI and value is the
1155 absolute path to the existing file in recipe space (if any).
1156 """
1157 import oe.recipeutils
1158 from oe.patch import GitApplyTree
1159 updated = OrderedDict()
1160 added = OrderedDict()
1161 seqpatch_re = re.compile('^([0-9]{4}-)?(.+)')
1162
1163 existing_patches = dict((os.path.basename(path), path) for path in
1164 oe.recipeutils.get_recipe_patches(rd))
1165
1166 # Generate patches from Git, exclude local files directory
1167 patch_pathspec = _git_exclude_path(srctree, 'oe-local-files')
1168 GitApplyTree.extractPatches(srctree, start_rev, destdir, patch_pathspec)
1169
1170 new_patches = sorted(os.listdir(destdir))
1171 for new_patch in new_patches:
1172 # Strip numbering from patch names. If it's a git sequence named patch,
1173 # the numbers might not match up since we are starting from a different
1174 # revision This does assume that people are using unique shortlog
1175 # values, but they ought to be anyway...
1176 new_basename = seqpatch_re.match(new_patch).group(2)
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001177 match_name = None
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001178 for old_patch in existing_patches:
1179 old_basename = seqpatch_re.match(old_patch).group(2)
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001180 old_basename_splitext = os.path.splitext(old_basename)
1181 if old_basename.endswith(('.gz', '.bz2', '.Z')) and old_basename_splitext[0] == new_basename:
1182 old_patch_noext = os.path.splitext(old_patch)[0]
1183 match_name = old_patch_noext
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001184 break
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001185 elif new_basename == old_basename:
1186 match_name = old_patch
1187 break
1188 if match_name:
1189 # Rename patch files
1190 if new_patch != match_name:
1191 os.rename(os.path.join(destdir, new_patch),
1192 os.path.join(destdir, match_name))
1193 # Need to pop it off the list now before checking changed_revs
1194 oldpath = existing_patches.pop(old_patch)
1195 if changed_revs is not None:
1196 # Avoid updating patches that have not actually changed
1197 with open(os.path.join(destdir, match_name), 'r') as f:
1198 firstlineitems = f.readline().split()
1199 # Looking for "From <hash>" line
1200 if len(firstlineitems) > 1 and len(firstlineitems[1]) == 40:
1201 if not firstlineitems[1] in changed_revs:
1202 continue
1203 # Recompress if necessary
1204 if oldpath.endswith(('.gz', '.Z')):
1205 bb.process.run(['gzip', match_name], cwd=destdir)
1206 if oldpath.endswith('.gz'):
1207 match_name += '.gz'
1208 else:
1209 match_name += '.Z'
1210 elif oldpath.endswith('.bz2'):
1211 bb.process.run(['bzip2', match_name], cwd=destdir)
1212 match_name += '.bz2'
1213 updated[match_name] = oldpath
1214 else:
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001215 added[new_patch] = None
1216 return (updated, added, existing_patches)
1217
1218
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001219def _create_kconfig_diff(srctree, rd, outfile):
1220 """Create a kconfig fragment"""
1221 # Only update config fragment if both config files exist
1222 orig_config = os.path.join(srctree, '.config.baseline')
1223 new_config = os.path.join(srctree, '.config.new')
1224 if os.path.exists(orig_config) and os.path.exists(new_config):
1225 cmd = ['diff', '--new-line-format=%L', '--old-line-format=',
1226 '--unchanged-line-format=', orig_config, new_config]
1227 pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE,
1228 stderr=subprocess.PIPE)
1229 stdout, stderr = pipe.communicate()
1230 if pipe.returncode == 1:
1231 logger.info("Updating config fragment %s" % outfile)
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001232 with open(outfile, 'wb') as fobj:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001233 fobj.write(stdout)
1234 elif pipe.returncode == 0:
1235 logger.info("Would remove config fragment %s" % outfile)
1236 if os.path.exists(outfile):
1237 # Remove fragment file in case of empty diff
1238 logger.info("Removing config fragment %s" % outfile)
1239 os.unlink(outfile)
1240 else:
1241 raise bb.process.ExecutionError(cmd, pipe.returncode, stdout, stderr)
1242 return True
1243 return False
1244
1245
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001246def _export_local_files(srctree, rd, destdir):
1247 """Copy local files from srctree to given location.
1248 Returns three-tuple of dicts:
1249 1. updated - files that already exist in SRCURI
1250 2. added - new files files that don't exist in SRCURI
1251 3 removed - files that exist in SRCURI but not in exported files
1252 In each dict the key is the 'basepath' of the URI and value is the
1253 absolute path to the existing file in recipe space (if any).
1254 """
1255 import oe.recipeutils
1256
1257 # Find out local files (SRC_URI files that exist in the "recipe space").
1258 # Local files that reside in srctree are not included in patch generation.
1259 # Instead they are directly copied over the original source files (in
1260 # recipe space).
1261 existing_files = oe.recipeutils.get_recipe_local_files(rd)
1262 new_set = None
1263 updated = OrderedDict()
1264 added = OrderedDict()
1265 removed = OrderedDict()
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001266 local_files_dir = os.path.join(srctree, 'oe-local-files')
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001267 git_files = _git_ls_tree(srctree)
1268 if 'oe-local-files' in git_files:
1269 # If tracked by Git, take the files from srctree HEAD. First get
1270 # the tree object of the directory
1271 tmp_index = os.path.join(srctree, '.git', 'index.tmp.devtool')
1272 tree = git_files['oe-local-files'][2]
1273 bb.process.run(['git', 'checkout', tree, '--', '.'], cwd=srctree,
1274 env=dict(os.environ, GIT_WORK_TREE=destdir,
1275 GIT_INDEX_FILE=tmp_index))
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001276 new_set = list(_git_ls_tree(srctree, tree, True).keys())
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001277 elif os.path.isdir(local_files_dir):
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001278 # If not tracked by Git, just copy from working copy
1279 new_set = _ls_tree(os.path.join(srctree, 'oe-local-files'))
1280 bb.process.run(['cp', '-ax',
1281 os.path.join(srctree, 'oe-local-files', '.'), destdir])
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001282 else:
1283 new_set = []
1284
1285 # Special handling for kernel config
1286 if bb.data.inherits_class('kernel-yocto', rd):
1287 fragment_fn = 'devtool-fragment.cfg'
1288 fragment_path = os.path.join(destdir, fragment_fn)
1289 if _create_kconfig_diff(srctree, rd, fragment_path):
1290 if os.path.exists(fragment_path):
1291 if fragment_fn not in new_set:
1292 new_set.append(fragment_fn)
1293 # Copy fragment to local-files
1294 if os.path.isdir(local_files_dir):
1295 shutil.copy2(fragment_path, local_files_dir)
1296 else:
1297 if fragment_fn in new_set:
1298 new_set.remove(fragment_fn)
1299 # Remove fragment from local-files
1300 if os.path.exists(os.path.join(local_files_dir, fragment_fn)):
1301 os.unlink(os.path.join(local_files_dir, fragment_fn))
1302
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001303 if new_set is not None:
1304 for fname in new_set:
1305 if fname in existing_files:
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001306 origpath = existing_files.pop(fname)
1307 workpath = os.path.join(local_files_dir, fname)
1308 if not filecmp.cmp(origpath, workpath):
1309 updated[fname] = origpath
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001310 elif fname != '.gitignore':
1311 added[fname] = None
1312
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001313 workdir = rd.getVar('WORKDIR')
1314 s = rd.getVar('S')
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001315 if not s.endswith(os.sep):
1316 s += os.sep
1317
1318 if workdir != s:
1319 # Handle files where subdir= was specified
1320 for fname in list(existing_files.keys()):
1321 # FIXME handle both subdir starting with BP and not?
1322 fworkpath = os.path.join(workdir, fname)
1323 if fworkpath.startswith(s):
1324 fpath = os.path.join(srctree, os.path.relpath(fworkpath, s))
1325 if os.path.exists(fpath):
1326 origpath = existing_files.pop(fname)
1327 if not filecmp.cmp(origpath, fpath):
1328 updated[fpath] = origpath
1329
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001330 removed = existing_files
1331 return (updated, added, removed)
1332
1333
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001334def _determine_files_dir(rd):
1335 """Determine the appropriate files directory for a recipe"""
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001336 recipedir = rd.getVar('FILE_DIRNAME')
1337 for entry in rd.getVar('FILESPATH').split(':'):
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001338 relpth = os.path.relpath(entry, recipedir)
1339 if not os.sep in relpth:
1340 # One (or zero) levels below only, so we don't put anything in machine-specific directories
1341 if os.path.isdir(entry):
1342 return entry
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001343 return os.path.join(recipedir, rd.getVar('BPN'))
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001344
1345
1346def _update_recipe_srcrev(srctree, rd, appendlayerdir, wildcard_version, no_remove):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001347 """Implement the 'srcrev' mode of update-recipe"""
1348 import bb
1349 import oe.recipeutils
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001350
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001351 recipefile = rd.getVar('FILE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001352 logger.info('Updating SRCREV in recipe %s' % os.path.basename(recipefile))
1353
1354 # Get HEAD revision
1355 try:
1356 stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree)
1357 except bb.process.ExecutionError as err:
1358 raise DevtoolError('Failed to get HEAD revision in %s: %s' %
1359 (srctree, err))
1360 srcrev = stdout.strip()
1361 if len(srcrev) != 40:
1362 raise DevtoolError('Invalid hash returned by git: %s' % stdout)
1363
1364 destpath = None
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001365 remove_files = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001366 patchfields = {}
1367 patchfields['SRCREV'] = srcrev
1368 orig_src_uri = rd.getVar('SRC_URI', False) or ''
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001369 srcuri = orig_src_uri.split()
1370 tempdir = tempfile.mkdtemp(prefix='devtool')
1371 update_srcuri = False
1372 try:
1373 local_files_dir = tempfile.mkdtemp(dir=tempdir)
1374 upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001375 if not no_remove:
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001376 # Find list of existing patches in recipe file
1377 patches_dir = tempfile.mkdtemp(dir=tempdir)
1378 old_srcrev = (rd.getVar('SRCREV', False) or '')
1379 upd_p, new_p, del_p = _export_patches(srctree, rd, old_srcrev,
1380 patches_dir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001381
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001382 # Remove deleted local files and "overlapping" patches
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001383 remove_files = list(del_f.values()) + list(upd_p.values())
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001384 if remove_files:
1385 removedentries = _remove_file_entries(srcuri, remove_files)[0]
1386 update_srcuri = True
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001387
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001388 if appendlayerdir:
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001389 files = dict((os.path.join(local_files_dir, key), val) for
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001390 key, val in list(upd_f.items()) + list(new_f.items()))
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001391 removevalues = {}
1392 if update_srcuri:
1393 removevalues = {'SRC_URI': removedentries}
1394 patchfields['SRC_URI'] = '\\\n '.join(srcuri)
1395 _, destpath = oe.recipeutils.bbappend_recipe(
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001396 rd, appendlayerdir, files, wildcardver=wildcard_version,
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001397 extralines=patchfields, removevalues=removevalues)
1398 else:
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001399 files_dir = _determine_files_dir(rd)
1400 for basepath, path in upd_f.items():
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001401 logger.info('Updating file %s' % basepath)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001402 if os.path.isabs(basepath):
1403 # Original file (probably with subdir pointing inside source tree)
1404 # so we do not want to move it, just copy
1405 _copy_file(basepath, path)
1406 else:
1407 _move_file(os.path.join(local_files_dir, basepath), path)
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001408 update_srcuri= True
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001409 for basepath, path in new_f.items():
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001410 logger.info('Adding new file %s' % basepath)
1411 _move_file(os.path.join(local_files_dir, basepath),
1412 os.path.join(files_dir, basepath))
1413 srcuri.append('file://%s' % basepath)
1414 update_srcuri = True
1415 if update_srcuri:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001416 patchfields['SRC_URI'] = ' '.join(srcuri)
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001417 oe.recipeutils.patch_recipe(rd, recipefile, patchfields)
1418 finally:
1419 shutil.rmtree(tempdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001420 if not 'git://' in orig_src_uri:
1421 logger.info('You will need to update SRC_URI within the recipe to '
1422 'point to a git repository where you have pushed your '
1423 'changes')
1424
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001425 _remove_source_files(appendlayerdir, remove_files, destpath)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001426 return True
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001427
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001428def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, initial_rev):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001429 """Implement the 'patch' mode of update-recipe"""
1430 import bb
1431 import oe.recipeutils
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001432
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001433 recipefile = rd.getVar('FILE')
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001434 append = workspace[recipename]['bbappend']
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001435 if not os.path.exists(append):
1436 raise DevtoolError('unable to find workspace bbappend for recipe %s' %
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001437 recipename)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001438
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001439 initial_rev, update_rev, changed_revs = _get_patchset_revs(srctree, append, initial_rev)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001440 if not initial_rev:
1441 raise DevtoolError('Unable to find initial revision - please specify '
1442 'it with --initial-rev')
1443
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001444 dl_dir = rd.getVar('DL_DIR')
1445 if not dl_dir.endswith('/'):
1446 dl_dir += '/'
1447
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001448 tempdir = tempfile.mkdtemp(prefix='devtool')
1449 try:
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001450 local_files_dir = tempfile.mkdtemp(dir=tempdir)
1451 upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001452
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001453 remove_files = []
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001454 if not no_remove:
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001455 # Get all patches from source tree and check if any should be removed
1456 all_patches_dir = tempfile.mkdtemp(dir=tempdir)
1457 upd_p, new_p, del_p = _export_patches(srctree, rd, initial_rev,
1458 all_patches_dir)
1459 # Remove deleted local files and patches
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001460 remove_files = list(del_f.values()) + list(del_p.values())
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001461
1462 # Get updated patches from source tree
1463 patches_dir = tempfile.mkdtemp(dir=tempdir)
1464 upd_p, new_p, del_p = _export_patches(srctree, rd, update_rev,
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001465 patches_dir, changed_revs)
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001466 updatefiles = False
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001467 updaterecipe = False
1468 destpath = None
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001469 srcuri = (rd.getVar('SRC_URI', False) or '').split()
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001470 if appendlayerdir:
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001471 files = dict((os.path.join(local_files_dir, key), val) for
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001472 key, val in list(upd_f.items()) + list(new_f.items()))
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001473 files.update(dict((os.path.join(patches_dir, key), val) for
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001474 key, val in list(upd_p.items()) + list(new_p.items())))
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001475 if files or remove_files:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001476 removevalues = None
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001477 if remove_files:
1478 removedentries, remaining = _remove_file_entries(
1479 srcuri, remove_files)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001480 if removedentries or remaining:
1481 remaining = ['file://' + os.path.basename(item) for
1482 item in remaining]
1483 removevalues = {'SRC_URI': removedentries + remaining}
1484 _, destpath = oe.recipeutils.bbappend_recipe(
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001485 rd, appendlayerdir, files,
1486 wildcardver=wildcard_version,
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001487 removevalues=removevalues)
1488 else:
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001489 logger.info('No patches or local source files needed updating')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001490 else:
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001491 # Update existing files
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001492 files_dir = _determine_files_dir(rd)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001493 for basepath, path in upd_f.items():
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001494 logger.info('Updating file %s' % basepath)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001495 if os.path.isabs(basepath):
1496 # Original file (probably with subdir pointing inside source tree)
1497 # so we do not want to move it, just copy
1498 _copy_file(basepath, path)
1499 else:
1500 _move_file(os.path.join(local_files_dir, basepath), path)
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001501 updatefiles = True
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001502 for basepath, path in upd_p.items():
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001503 patchfn = os.path.join(patches_dir, basepath)
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001504 if os.path.dirname(path) + '/' == dl_dir:
1505 # This is a a downloaded patch file - we now need to
1506 # replace the entry in SRC_URI with our local version
1507 logger.info('Replacing remote patch %s with updated local version' % basepath)
1508 path = os.path.join(files_dir, basepath)
1509 _replace_srcuri_entry(srcuri, basepath, 'file://%s' % basepath)
1510 updaterecipe = True
1511 else:
1512 logger.info('Updating patch %s' % basepath)
1513 logger.debug('Moving new patch %s to %s' % (patchfn, path))
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001514 _move_file(patchfn, path)
1515 updatefiles = True
1516 # Add any new files
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001517 for basepath, path in new_f.items():
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001518 logger.info('Adding new file %s' % basepath)
1519 _move_file(os.path.join(local_files_dir, basepath),
1520 os.path.join(files_dir, basepath))
1521 srcuri.append('file://%s' % basepath)
1522 updaterecipe = True
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001523 for basepath, path in new_p.items():
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001524 logger.info('Adding new patch %s' % basepath)
1525 _move_file(os.path.join(patches_dir, basepath),
1526 os.path.join(files_dir, basepath))
1527 srcuri.append('file://%s' % basepath)
1528 updaterecipe = True
1529 # Update recipe, if needed
1530 if _remove_file_entries(srcuri, remove_files)[0]:
1531 updaterecipe = True
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001532 if updaterecipe:
1533 logger.info('Updating recipe %s' % os.path.basename(recipefile))
1534 oe.recipeutils.patch_recipe(rd, recipefile,
1535 {'SRC_URI': ' '.join(srcuri)})
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001536 elif not updatefiles:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001537 # Neither patches nor recipe were updated
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001538 logger.info('No patches or files need updating')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001539 return False
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001540 finally:
1541 shutil.rmtree(tempdir)
1542
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001543 _remove_source_files(appendlayerdir, remove_files, destpath)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001544 return True
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001545
1546def _guess_recipe_update_mode(srctree, rdata):
1547 """Guess the recipe update mode to use"""
1548 src_uri = (rdata.getVar('SRC_URI', False) or '').split()
1549 git_uris = [uri for uri in src_uri if uri.startswith('git://')]
1550 if not git_uris:
1551 return 'patch'
1552 # Just use the first URI for now
1553 uri = git_uris[0]
1554 # Check remote branch
1555 params = bb.fetch.decodeurl(uri)[5]
1556 upstr_branch = params['branch'] if 'branch' in params else 'master'
1557 # Check if current branch HEAD is found in upstream branch
1558 stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree)
1559 head_rev = stdout.rstrip()
1560 stdout, _ = bb.process.run('git branch -r --contains %s' % head_rev,
1561 cwd=srctree)
1562 remote_brs = [branch.strip() for branch in stdout.splitlines()]
1563 if 'origin/' + upstr_branch in remote_brs:
1564 return 'srcrev'
1565
1566 return 'patch'
1567
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001568def _update_recipe(recipename, workspace, rd, mode, appendlayerdir, wildcard_version, no_remove, initial_rev):
1569 srctree = workspace[recipename]['srctree']
1570 if mode == 'auto':
1571 mode = _guess_recipe_update_mode(srctree, rd)
1572
1573 if mode == 'srcrev':
1574 updated = _update_recipe_srcrev(srctree, rd, appendlayerdir, wildcard_version, no_remove)
1575 elif mode == 'patch':
1576 updated = _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, initial_rev)
1577 else:
1578 raise DevtoolError('update_recipe: invalid mode %s' % mode)
1579 return updated
1580
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001581def update_recipe(args, config, basepath, workspace):
1582 """Entry point for the devtool 'update-recipe' subcommand"""
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001583 check_workspace_recipe(workspace, args.recipename)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001584
1585 if args.append:
1586 if not os.path.exists(args.append):
1587 raise DevtoolError('bbappend destination layer directory "%s" '
1588 'does not exist' % args.append)
1589 if not os.path.exists(os.path.join(args.append, 'conf', 'layer.conf')):
1590 raise DevtoolError('conf/layer.conf not found in bbappend '
1591 'destination layer "%s"' % args.append)
1592
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001593 tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001594 try:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001595
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001596 rd = parse_recipe(config, tinfoil, args.recipename, True)
1597 if not rd:
1598 return 1
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001599
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001600 updated = _update_recipe(args.recipename, workspace, rd, args.mode, args.append, args.wildcard_version, args.no_remove, args.initial_rev)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001601
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001602 if updated:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001603 rf = rd.getVar('FILE')
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001604 if rf.startswith(config.workspace_path):
1605 logger.warn('Recipe file %s has been updated but is inside the workspace - you will need to move it (and any associated files next to it) out to the desired layer before using "devtool reset" in order to keep any changes' % rf)
1606 finally:
1607 tinfoil.shutdown()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001608
1609 return 0
1610
1611
1612def status(args, config, basepath, workspace):
1613 """Entry point for the devtool 'status' subcommand"""
1614 if workspace:
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001615 for recipe, value in workspace.items():
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001616 recipefile = value['recipefile']
1617 if recipefile:
1618 recipestr = ' (%s)' % recipefile
1619 else:
1620 recipestr = ''
1621 print("%s: %s%s" % (recipe, value['srctree'], recipestr))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001622 else:
1623 logger.info('No recipes currently in your workspace - you can use "devtool modify" to work on an existing recipe or "devtool add" to add a new one')
1624 return 0
1625
1626
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001627def _reset(recipes, no_clean, config, basepath, workspace):
1628 """Reset one or more recipes"""
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001629
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001630 if recipes and not no_clean:
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001631 if len(recipes) == 1:
1632 logger.info('Cleaning sysroot for recipe %s...' % recipes[0])
1633 else:
1634 logger.info('Cleaning sysroot for recipes %s...' % ', '.join(recipes))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001635 # If the recipe file itself was created in the workspace, and
1636 # it uses BBCLASSEXTEND, then we need to also clean the other
1637 # variants
1638 targets = []
1639 for recipe in recipes:
1640 targets.append(recipe)
1641 recipefile = workspace[recipe]['recipefile']
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001642 if recipefile and os.path.exists(recipefile):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001643 targets.extend(get_bbclassextend_targets(recipefile, recipe))
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001644 try:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001645 exec_build_env_command(config.init_path, basepath, 'bitbake -c clean %s' % ' '.join(targets))
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001646 except bb.process.ExecutionError as e:
1647 raise DevtoolError('Command \'%s\' failed, output:\n%s\nIf you '
1648 'wish, you may specify -n/--no-clean to '
1649 'skip running this command when resetting' %
1650 (e.command, e.stdout))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001651
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001652 for pn in recipes:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001653 _check_preserve(config, pn)
1654
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001655 preservepath = os.path.join(config.workspace_path, 'attic', pn, pn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001656 def preservedir(origdir):
1657 if os.path.exists(origdir):
1658 for root, dirs, files in os.walk(origdir):
1659 for fn in files:
1660 logger.warn('Preserving %s in %s' % (fn, preservepath))
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001661 _move_file(os.path.join(origdir, fn),
1662 os.path.join(preservepath, fn))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001663 for dn in dirs:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001664 preservedir(os.path.join(root, dn))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001665 os.rmdir(origdir)
1666
1667 preservedir(os.path.join(config.workspace_path, 'recipes', pn))
1668 # We don't automatically create this dir next to appends, but the user can
1669 preservedir(os.path.join(config.workspace_path, 'appends', pn))
1670
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001671 srctree = workspace[pn]['srctree']
1672 if os.path.isdir(srctree):
1673 if os.listdir(srctree):
1674 # We don't want to risk wiping out any work in progress
1675 logger.info('Leaving source tree %s as-is; if you no '
1676 'longer need it then please delete it manually'
1677 % srctree)
1678 else:
1679 # This is unlikely, but if it's empty we can just remove it
1680 os.rmdir(srctree)
1681
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001682
1683def reset(args, config, basepath, workspace):
1684 """Entry point for the devtool 'reset' subcommand"""
1685 import bb
1686 if args.recipename:
1687 if args.all:
1688 raise DevtoolError("Recipe cannot be specified if -a/--all is used")
1689 else:
1690 for recipe in args.recipename:
1691 check_workspace_recipe(workspace, recipe, checksrc=False)
1692 elif not args.all:
1693 raise DevtoolError("Recipe must be specified, or specify -a/--all to "
1694 "reset all recipes")
1695 if args.all:
1696 recipes = list(workspace.keys())
1697 else:
1698 recipes = args.recipename
1699
1700 _reset(recipes, args.no_clean, config, basepath, workspace)
1701
1702 return 0
1703
1704
1705def _get_layer(layername, d):
1706 """Determine the base layer path for the specified layer name/path"""
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001707 layerdirs = d.getVar('BBLAYERS').split()
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001708 layers = {os.path.basename(p): p for p in layerdirs}
1709 # Provide some shortcuts
1710 if layername.lower() in ['oe-core', 'openembedded-core']:
1711 layerdir = layers.get('meta', None)
1712 else:
1713 layerdir = layers.get(layername, None)
1714 if layerdir:
1715 layerdir = os.path.abspath(layerdir)
1716 return layerdir or layername
1717
1718def finish(args, config, basepath, workspace):
1719 """Entry point for the devtool 'finish' subcommand"""
1720 import bb
1721 import oe.recipeutils
1722
1723 check_workspace_recipe(workspace, args.recipename)
1724
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001725 no_clean = False
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001726 tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
1727 try:
1728 rd = parse_recipe(config, tinfoil, args.recipename, True)
1729 if not rd:
1730 return 1
1731
1732 destlayerdir = _get_layer(args.destination, tinfoil.config_data)
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001733 origlayerdir = oe.recipeutils.find_layerdir(rd.getVar('FILE'))
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001734
1735 if not os.path.isdir(destlayerdir):
1736 raise DevtoolError('Unable to find layer or directory matching "%s"' % args.destination)
1737
1738 if os.path.abspath(destlayerdir) == config.workspace_path:
1739 raise DevtoolError('"%s" specifies the workspace layer - that is not a valid destination' % args.destination)
1740
1741 # If it's an upgrade, grab the original path
1742 origpath = None
1743 origfilelist = None
1744 append = workspace[args.recipename]['bbappend']
1745 with open(append, 'r') as f:
1746 for line in f:
1747 if line.startswith('# original_path:'):
1748 origpath = line.split(':')[1].strip()
1749 elif line.startswith('# original_files:'):
1750 origfilelist = line.split(':')[1].split()
1751
1752 if origlayerdir == config.workspace_path:
1753 # Recipe file itself is in workspace, update it there first
1754 appendlayerdir = None
1755 origrelpath = None
1756 if origpath:
1757 origlayerpath = oe.recipeutils.find_layerdir(origpath)
1758 if origlayerpath:
1759 origrelpath = os.path.relpath(origpath, origlayerpath)
1760 destpath = oe.recipeutils.get_bbfile_path(rd, destlayerdir, origrelpath)
1761 if not destpath:
1762 raise DevtoolError("Unable to determine destination layer path - check that %s specifies an actual layer and %s/conf/layer.conf specifies BBFILES. You may also need to specify a more complete path." % (args.destination, destlayerdir))
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001763 # Warn if the layer isn't in bblayers.conf (the code to create a bbappend will do this in other cases)
1764 layerdirs = [os.path.abspath(layerdir) for layerdir in rd.getVar('BBLAYERS').split()]
1765 if not os.path.abspath(destlayerdir) in layerdirs:
1766 bb.warn('Specified destination layer is not currently enabled in bblayers.conf, so the %s recipe will now be unavailable in your current configuration until you add the layer there' % args.recipename)
1767
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001768 elif destlayerdir == origlayerdir:
1769 # Same layer, update the original recipe
1770 appendlayerdir = None
1771 destpath = None
1772 else:
1773 # Create/update a bbappend in the specified layer
1774 appendlayerdir = destlayerdir
1775 destpath = None
1776
1777 # Remove any old files in the case of an upgrade
1778 if origpath and origfilelist and oe.recipeutils.find_layerdir(origpath) == oe.recipeutils.find_layerdir(destlayerdir):
1779 for fn in origfilelist:
1780 fnp = os.path.join(origpath, fn)
1781 try:
1782 os.remove(fnp)
1783 except FileNotFoundError:
1784 pass
1785
1786 # Actually update the recipe / bbappend
1787 _update_recipe(args.recipename, workspace, rd, args.mode, appendlayerdir, wildcard_version=True, no_remove=False, initial_rev=args.initial_rev)
1788
1789 if origlayerdir == config.workspace_path and destpath:
1790 # Recipe file itself is in the workspace - need to move it and any
1791 # associated files to the specified layer
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001792 no_clean = True
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001793 logger.info('Moving recipe file to %s' % destpath)
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001794 recipedir = os.path.dirname(rd.getVar('FILE'))
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001795 for root, _, files in os.walk(recipedir):
1796 for fn in files:
1797 srcpath = os.path.join(root, fn)
1798 relpth = os.path.relpath(os.path.dirname(srcpath), recipedir)
1799 destdir = os.path.abspath(os.path.join(destpath, relpth))
1800 bb.utils.mkdirhier(destdir)
1801 shutil.move(srcpath, os.path.join(destdir, fn))
1802
1803 finally:
1804 tinfoil.shutdown()
1805
1806 # Everything else has succeeded, we can now reset
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001807 _reset([args.recipename], no_clean=no_clean, config=config, basepath=basepath, workspace=workspace)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001808
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001809 return 0
1810
1811
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001812def get_default_srctree(config, recipename=''):
1813 """Get the default srctree path"""
1814 srctreeparent = config.get('General', 'default_source_parent_dir', config.workspace_path)
1815 if recipename:
1816 return os.path.join(srctreeparent, 'sources', recipename)
1817 else:
1818 return os.path.join(srctreeparent, 'sources')
1819
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001820def register_commands(subparsers, context):
1821 """Register devtool subcommands from this plugin"""
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001822
1823 defsrctree = get_default_srctree(context.config)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001824 parser_add = subparsers.add_parser('add', help='Add a new recipe',
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001825 description='Adds a new recipe to the workspace to build a specified source tree. Can optionally fetch a remote URI and unpack it to create the source tree.',
1826 group='starting', order=100)
1827 parser_add.add_argument('recipename', nargs='?', help='Name for new recipe to add (just name - no version, path or extension). If not specified, will attempt to auto-detect it.')
1828 parser_add.add_argument('srctree', nargs='?', help='Path to external source tree. If not specified, a subdirectory of %s will be used.' % defsrctree)
1829 parser_add.add_argument('fetchuri', nargs='?', help='Fetch the specified URI and extract it to create the source tree')
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001830 group = parser_add.add_mutually_exclusive_group()
1831 group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true")
1832 group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001833 parser_add.add_argument('--fetch', '-f', help='Fetch the specified URI and extract it to create the source tree (deprecated - pass as positional argument instead)', metavar='URI')
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001834 parser_add.add_argument('--fetch-dev', help='For npm, also fetch devDependencies', action="store_true")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001835 parser_add.add_argument('--version', '-V', help='Version to use within recipe (PV)')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001836 parser_add.add_argument('--no-git', '-g', help='If fetching source, do not set up source tree as a git repository', action="store_true")
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001837 parser_add.add_argument('--autorev', '-a', help='When fetching from a git repository, set SRCREV in the recipe to a floating revision instead of fixed', action="store_true")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001838 parser_add.add_argument('--binary', '-b', help='Treat the source tree as something that should be installed verbatim (no compilation, same directory structure). Useful with binary packages e.g. RPMs.', action='store_true')
1839 parser_add.add_argument('--also-native', help='Also add native variant (i.e. support building recipe for the build host as well as the target machine)', action='store_true')
1840 parser_add.add_argument('--src-subdir', help='Specify subdirectory within source tree to use', metavar='SUBDIR')
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001841 parser_add.set_defaults(func=add, fixed_setup=context.fixed_setup)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001842
1843 parser_modify = subparsers.add_parser('modify', help='Modify the source for an existing recipe',
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001844 description='Sets up the build environment to modify the source for an existing recipe. The default behaviour is to extract the source being fetched by the recipe into a git tree so you can work on it; alternatively if you already have your own pre-prepared source tree you can specify -n/--no-extract.',
1845 group='starting', order=90)
1846 parser_modify.add_argument('recipename', help='Name of existing recipe to edit (just name - no version, path or extension)')
1847 parser_modify.add_argument('srctree', nargs='?', help='Path to external source tree. If not specified, a subdirectory of %s will be used.' % defsrctree)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001848 parser_modify.add_argument('--wildcard', '-w', action="store_true", help='Use wildcard for unversioned bbappend')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001849 group = parser_modify.add_mutually_exclusive_group()
1850 group.add_argument('--extract', '-x', action="store_true", help='Extract source for recipe (default)')
1851 group.add_argument('--no-extract', '-n', action="store_true", help='Do not extract source, expect it to exist')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001852 group = parser_modify.add_mutually_exclusive_group()
1853 group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true")
1854 group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001855 parser_modify.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (when not using -n/--no-extract) (default "%(default)s")')
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001856 parser_modify.add_argument('--keep-temp', help='Keep temporary directory (for debugging)', action="store_true")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001857 parser_modify.set_defaults(func=modify)
1858
1859 parser_extract = subparsers.add_parser('extract', help='Extract the source for an existing recipe',
1860 description='Extracts the source for an existing recipe',
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001861 group='advanced')
1862 parser_extract.add_argument('recipename', help='Name of recipe to extract the source for')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001863 parser_extract.add_argument('srctree', help='Path to where to extract the source tree')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001864 parser_extract.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (default "%(default)s")')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001865 parser_extract.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001866 parser_extract.set_defaults(func=extract, no_workspace=True)
1867
1868 parser_sync = subparsers.add_parser('sync', help='Synchronize the source tree for an existing recipe',
1869 description='Synchronize the previously extracted source tree for an existing recipe',
1870 formatter_class=argparse.ArgumentDefaultsHelpFormatter,
1871 group='advanced')
1872 parser_sync.add_argument('recipename', help='Name of recipe to sync the source for')
1873 parser_sync.add_argument('srctree', help='Path to the source tree')
1874 parser_sync.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout')
1875 parser_sync.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
1876 parser_sync.set_defaults(func=sync)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001877
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001878 parser_rename = subparsers.add_parser('rename', help='Rename a recipe file in the workspace',
1879 description='Renames the recipe file for a recipe in the workspace, changing the name or version part or both, ensuring that all references within the workspace are updated at the same time. Only works when the recipe file itself is in the workspace, e.g. after devtool add. Particularly useful when devtool add did not automatically determine the correct name.',
1880 group='working', order=10)
1881 parser_rename.add_argument('recipename', help='Current name of recipe to rename')
1882 parser_rename.add_argument('newname', nargs='?', help='New name for recipe (optional, not needed if you only want to change the version)')
1883 parser_rename.add_argument('--version', '-V', help='Change the version (NOTE: this does not change the version fetched by the recipe, just the version in the recipe file name)')
1884 parser_rename.add_argument('--no-srctree', '-s', action='store_true', help='Do not rename the source tree directory (if the default source tree path has been used) - keeping the old name may be desirable if there are internal/other external references to this path')
1885 parser_rename.set_defaults(func=rename)
1886
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001887 parser_update_recipe = subparsers.add_parser('update-recipe', help='Apply changes from external source tree to recipe',
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001888 description='Applies changes from external source tree to a recipe (updating/adding/removing patches as necessary, or by updating SRCREV). Note that these changes need to have been committed to the git repository in order to be recognised.',
1889 group='working', order=-90)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001890 parser_update_recipe.add_argument('recipename', help='Name of recipe to update')
1891 parser_update_recipe.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001892 parser_update_recipe.add_argument('--initial-rev', help='Override starting revision for patches')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001893 parser_update_recipe.add_argument('--append', '-a', help='Write changes to a bbappend in the specified layer instead of the recipe', metavar='LAYERDIR')
1894 parser_update_recipe.add_argument('--wildcard-version', '-w', help='In conjunction with -a/--append, use a wildcard to make the bbappend apply to any recipe version', action='store_true')
1895 parser_update_recipe.add_argument('--no-remove', '-n', action="store_true", help='Don\'t remove patches, only add or update')
1896 parser_update_recipe.set_defaults(func=update_recipe)
1897
1898 parser_status = subparsers.add_parser('status', help='Show workspace status',
1899 description='Lists recipes currently in your workspace and the paths to their respective external source trees',
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001900 group='info', order=100)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001901 parser_status.set_defaults(func=status)
1902
1903 parser_reset = subparsers.add_parser('reset', help='Remove a recipe from your workspace',
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001904 description='Removes the specified recipe(s) from your workspace (resetting its state back to that defined by the metadata).',
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001905 group='working', order=-100)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001906 parser_reset.add_argument('recipename', nargs='*', help='Recipe to reset')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001907 parser_reset.add_argument('--all', '-a', action="store_true", help='Reset all recipes (clear workspace)')
1908 parser_reset.add_argument('--no-clean', '-n', action="store_true", help='Don\'t clean the sysroot to remove recipe output')
1909 parser_reset.set_defaults(func=reset)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001910
1911 parser_finish = subparsers.add_parser('finish', help='Finish working on a recipe in your workspace',
1912 description='Pushes any committed changes to the specified recipe to the specified layer and removes it from your workspace. Roughly equivalent to an update-recipe followed by reset, except the update-recipe step will do the "right thing" depending on the recipe and the destination layer specified.',
1913 group='working', order=-100)
1914 parser_finish.add_argument('recipename', help='Recipe to finish')
1915 parser_finish.add_argument('destination', help='Layer/path to put recipe into. Can be the name of a layer configured in your bblayers.conf, the path to the base of a layer, or a partial path inside a layer. %(prog)s will attempt to complete the path based on the layer\'s structure.')
1916 parser_finish.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE')
1917 parser_finish.add_argument('--initial-rev', help='Override starting revision for patches')
1918 parser_finish.set_defaults(func=finish)