blob: b74a60d001173557e0a1b8f1f1346bbc95a931a2 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# Development tool - standard commands plugin
2#
Brad Bishopd7bf8c12018-02-25 22:55:05 -05003# Copyright (C) 2014-2017 Intel Corporation
Patrick Williamsc124f4f2015-09-15 14:41:29 -05004#
Brad Bishopc342db32019-05-15 21:57:59 -04005# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -05006#
Patrick Williamsc124f4f2015-09-15 14:41:29 -05007"""Devtool standard plugins"""
8
9import os
10import sys
11import re
12import shutil
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050013import subprocess
Patrick Williamsc124f4f2015-09-15 14:41:29 -050014import tempfile
15import logging
16import argparse
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050017import argparse_oe
Patrick Williamsc124f4f2015-09-15 14:41:29 -050018import scriptutils
19import errno
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050020import glob
Patrick Williamsc0f7c042017-02-23 20:41:17 -060021import filecmp
Patrick Williamsf1e5d692016-03-30 15:21:19 -050022from collections import OrderedDict
Brad Bishop316dfdd2018-06-25 12:45:53 -040023from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, use_external_build, setup_git_repo, recipe_to_append, get_bbclassextend_targets, update_unlockedsigs, check_prerelease_version, check_git_repo_dirty, check_git_repo_op, DevtoolError
Patrick Williamsc124f4f2015-09-15 14:41:29 -050024from devtool import parse_recipe
25
26logger = logging.getLogger('devtool')
27
Brad Bishop316dfdd2018-06-25 12:45:53 -040028override_branch_prefix = 'devtool-override-'
29
Patrick Williamsc124f4f2015-09-15 14:41:29 -050030
31def add(args, config, basepath, workspace):
32 """Entry point for the devtool 'add' subcommand"""
33 import bb
34 import oe.recipeutils
35
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050036 if not args.recipename and not args.srctree and not args.fetch and not args.fetchuri:
37 raise argparse_oe.ArgumentUsageError('At least one of recipename, srctree, fetchuri or -f/--fetch must be specified', 'add')
Patrick Williamsc124f4f2015-09-15 14:41:29 -050038
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050039 # These are positional arguments, but because we're nice, allow
40 # specifying e.g. source tree without name, or fetch URI without name or
41 # source tree (if we can detect that that is what the user meant)
Patrick Williamsc0f7c042017-02-23 20:41:17 -060042 if scriptutils.is_src_url(args.recipename):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050043 if not args.fetchuri:
44 if args.fetch:
45 raise DevtoolError('URI specified as positional argument as well as -f/--fetch')
46 args.fetchuri = args.recipename
47 args.recipename = ''
Patrick Williamsc0f7c042017-02-23 20:41:17 -060048 elif scriptutils.is_src_url(args.srctree):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050049 if not args.fetchuri:
50 if args.fetch:
51 raise DevtoolError('URI specified as positional argument as well as -f/--fetch')
52 args.fetchuri = args.srctree
53 args.srctree = ''
54 elif args.recipename and not args.srctree:
55 if os.sep in args.recipename:
56 args.srctree = args.recipename
57 args.recipename = None
58 elif os.path.isdir(args.recipename):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080059 logger.warning('Ambiguous argument "%s" - assuming you mean it to be the recipe name' % args.recipename)
Patrick Williamsc0f7c042017-02-23 20:41:17 -060060
Brad Bishopd7bf8c12018-02-25 22:55:05 -050061 if not args.fetchuri:
62 if args.srcrev:
63 raise DevtoolError('The -S/--srcrev option is only valid when fetching from an SCM repository')
64 if args.srcbranch:
65 raise DevtoolError('The -B/--srcbranch option is only valid when fetching from an SCM repository')
66
Patrick Williamsc0f7c042017-02-23 20:41:17 -060067 if args.srctree and os.path.isfile(args.srctree):
68 args.fetchuri = 'file://' + os.path.abspath(args.srctree)
69 args.srctree = ''
Patrick Williamsc124f4f2015-09-15 14:41:29 -050070
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050071 if args.fetch:
72 if args.fetchuri:
73 raise DevtoolError('URI specified as positional argument as well as -f/--fetch')
74 else:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080075 logger.warning('-f/--fetch option is deprecated - you can now simply specify the URL to fetch as a positional argument instead')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050076 args.fetchuri = args.fetch
Patrick Williamsf1e5d692016-03-30 15:21:19 -050077
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050078 if args.recipename:
79 if args.recipename in workspace:
80 raise DevtoolError("recipe %s is already in your workspace" %
81 args.recipename)
82 reason = oe.recipeutils.validate_pn(args.recipename)
83 if reason:
84 raise DevtoolError(reason)
85
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050086 if args.srctree:
87 srctree = os.path.abspath(args.srctree)
88 srctreeparent = None
89 tmpsrcdir = None
90 else:
91 srctree = None
92 srctreeparent = get_default_srctree(config)
93 bb.utils.mkdirhier(srctreeparent)
94 tmpsrcdir = tempfile.mkdtemp(prefix='devtoolsrc', dir=srctreeparent)
95
96 if srctree and os.path.exists(srctree):
97 if args.fetchuri:
Patrick Williamsc124f4f2015-09-15 14:41:29 -050098 if not os.path.isdir(srctree):
99 raise DevtoolError("Cannot fetch into source tree path %s as "
100 "it exists and is not a directory" %
101 srctree)
102 elif os.listdir(srctree):
103 raise DevtoolError("Cannot fetch into source tree path %s as "
104 "it already exists and is non-empty" %
105 srctree)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500106 elif not args.fetchuri:
107 if args.srctree:
108 raise DevtoolError("Specified source tree %s could not be found" %
109 args.srctree)
110 elif srctree:
111 raise DevtoolError("No source tree exists at default path %s - "
112 "either create and populate this directory, "
113 "or specify a path to a source tree, or a "
114 "URI to fetch source from" % srctree)
115 else:
116 raise DevtoolError("You must either specify a source tree "
117 "or a URI to fetch source from")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500118
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500119 if args.version:
120 if '_' in args.version or ' ' in args.version:
121 raise DevtoolError('Invalid version string "%s"' % args.version)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500122
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500123 if args.color == 'auto' and sys.stdout.isatty():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500124 color = 'always'
125 else:
126 color = args.color
127 extracmdopts = ''
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500128 if args.fetchuri:
129 source = args.fetchuri
130 if srctree:
131 extracmdopts += ' -x %s' % srctree
132 else:
133 extracmdopts += ' -x %s' % tmpsrcdir
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500134 else:
135 source = srctree
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500136 if args.recipename:
137 extracmdopts += ' -N %s' % args.recipename
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500138 if args.version:
139 extracmdopts += ' -V %s' % args.version
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500140 if args.binary:
141 extracmdopts += ' -b'
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500142 if args.also_native:
143 extracmdopts += ' --also-native'
144 if args.src_subdir:
145 extracmdopts += ' --src-subdir "%s"' % args.src_subdir
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600146 if args.autorev:
147 extracmdopts += ' -a'
Andrew Geissler82c905d2020-04-13 13:39:40 -0500148 if args.npm_dev:
149 extracmdopts += ' --npm-dev'
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500150 if args.mirrors:
151 extracmdopts += ' --mirrors'
152 if args.srcrev:
153 extracmdopts += ' --srcrev %s' % args.srcrev
154 if args.srcbranch:
155 extracmdopts += ' --srcbranch %s' % args.srcbranch
156 if args.provides:
157 extracmdopts += ' --provides %s' % args.provides
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500158
159 tempdir = tempfile.mkdtemp(prefix='devtool')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500160 try:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500161 try:
162 stdout, _ = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create --devtool -o %s \'%s\' %s' % (color, tempdir, source, extracmdopts), watch=True)
163 except bb.process.ExecutionError as e:
164 if e.exitcode == 15:
165 raise DevtoolError('Could not auto-determine recipe name, please specify it on the command line')
166 else:
167 raise DevtoolError('Command \'%s\' failed' % e.command)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500168
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500169 recipes = glob.glob(os.path.join(tempdir, '*.bb'))
170 if recipes:
171 recipename = os.path.splitext(os.path.basename(recipes[0]))[0].split('_')[0]
172 if recipename in workspace:
173 raise DevtoolError('A recipe with the same name as the one being created (%s) already exists in your workspace' % recipename)
174 recipedir = os.path.join(config.workspace_path, 'recipes', recipename)
175 bb.utils.mkdirhier(recipedir)
176 recipefile = os.path.join(recipedir, os.path.basename(recipes[0]))
177 appendfile = recipe_to_append(recipefile, config)
178 if os.path.exists(appendfile):
179 # This shouldn't be possible, but just in case
180 raise DevtoolError('A recipe with the same name as the one being created already exists in your workspace')
181 if os.path.exists(recipefile):
182 raise DevtoolError('A recipe file %s already exists in your workspace; this shouldn\'t be there - please delete it before continuing' % recipefile)
183 if tmpsrcdir:
184 srctree = os.path.join(srctreeparent, recipename)
185 if os.path.exists(tmpsrcdir):
186 if os.path.exists(srctree):
187 if os.path.isdir(srctree):
188 try:
189 os.rmdir(srctree)
190 except OSError as e:
191 if e.errno == errno.ENOTEMPTY:
192 raise DevtoolError('Source tree path %s already exists and is not empty' % srctree)
193 else:
194 raise
195 else:
196 raise DevtoolError('Source tree path %s already exists and is not a directory' % srctree)
197 logger.info('Using default source tree path %s' % srctree)
198 shutil.move(tmpsrcdir, srctree)
199 else:
200 raise DevtoolError('Couldn\'t find source tree created by recipetool')
201 bb.utils.mkdirhier(recipedir)
202 shutil.move(recipes[0], recipefile)
203 # Move any additional files created by recipetool
204 for fn in os.listdir(tempdir):
205 shutil.move(os.path.join(tempdir, fn), recipedir)
206 else:
207 raise DevtoolError('Command \'%s\' did not create any recipe file:\n%s' % (e.command, e.stdout))
208 attic_recipe = os.path.join(config.workspace_path, 'attic', recipename, os.path.basename(recipefile))
209 if os.path.exists(attic_recipe):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800210 logger.warning('A modified recipe from a previous invocation exists in %s - you may wish to move this over the top of the new recipe if you had changes in it that you want to continue with' % attic_recipe)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500211 finally:
212 if tmpsrcdir and os.path.exists(tmpsrcdir):
213 shutil.rmtree(tmpsrcdir)
214 shutil.rmtree(tempdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500215
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500216 for fn in os.listdir(recipedir):
217 _add_md5(config, recipename, os.path.join(recipedir, fn))
218
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500219 tinfoil = setup_tinfoil(config_only=True, basepath=basepath)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600220 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500221 try:
222 rd = tinfoil.parse_recipe_file(recipefile, False)
223 except Exception as e:
224 logger.error(str(e))
225 rd = None
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600226 if not rd:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500227 # Parsing failed. We just created this recipe and we shouldn't
228 # leave it in the workdir or it'll prevent bitbake from starting
229 movefn = '%s.parsefailed' % recipefile
230 logger.error('Parsing newly created recipe failed, moving recipe to %s for reference. If this looks to be caused by the recipe itself, please report this error.' % movefn)
231 shutil.move(recipefile, movefn)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600232 return 1
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500233
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600234 if args.fetchuri and not args.no_git:
235 setup_git_repo(srctree, args.version, 'devtool', d=tinfoil.config_data)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500236
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600237 initial_rev = None
238 if os.path.exists(os.path.join(srctree, '.git')):
239 (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree)
240 initial_rev = stdout.rstrip()
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500241
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600242 if args.src_subdir:
243 srctree = os.path.join(srctree, args.src_subdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500244
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600245 bb.utils.mkdirhier(os.path.dirname(appendfile))
246 with open(appendfile, 'w') as f:
247 f.write('inherit externalsrc\n')
248 f.write('EXTERNALSRC = "%s"\n' % srctree)
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500249
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600250 b_is_s = use_external_build(args.same_dir, args.no_same_dir, rd)
251 if b_is_s:
252 f.write('EXTERNALSRC_BUILD = "%s"\n' % srctree)
253 if initial_rev:
254 f.write('\n# initial_rev: %s\n' % initial_rev)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500255
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600256 if args.binary:
Patrick Williams213cb262021-08-07 19:21:33 -0500257 f.write('do_install:append() {\n')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600258 f.write(' rm -rf ${D}/.git\n')
259 f.write(' rm -f ${D}/singletask.lock\n')
260 f.write('}\n')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500261
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600262 if bb.data.inherits_class('npm', rd):
Patrick Williams213cb262021-08-07 19:21:33 -0500263 f.write('python do_configure:append() {\n')
Andrew Geissler82c905d2020-04-13 13:39:40 -0500264 f.write(' pkgdir = d.getVar("NPM_PACKAGE")\n')
265 f.write(' lockfile = os.path.join(pkgdir, "singletask.lock")\n')
266 f.write(' bb.utils.remove(lockfile)\n')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600267 f.write('}\n')
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500268
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500269 # Check if the new layer provides recipes whose priorities have been
270 # overriden by PREFERRED_PROVIDER.
271 recipe_name = rd.getVar('PN')
272 provides = rd.getVar('PROVIDES')
273 # Search every item defined in PROVIDES
274 for recipe_provided in provides.split():
275 preferred_provider = 'PREFERRED_PROVIDER_' + recipe_provided
276 current_pprovider = rd.getVar(preferred_provider)
277 if current_pprovider and current_pprovider != recipe_name:
278 if args.fixed_setup:
279 #if we are inside the eSDK add the new PREFERRED_PROVIDER in the workspace layer.conf
280 layerconf_file = os.path.join(config.workspace_path, "conf", "layer.conf")
281 with open(layerconf_file, 'a') as f:
282 f.write('%s = "%s"\n' % (preferred_provider, recipe_name))
283 else:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800284 logger.warning('Set \'%s\' in order to use the recipe' % preferred_provider)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500285 break
286
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600287 _add_md5(config, recipename, appendfile)
288
Brad Bishop316dfdd2018-06-25 12:45:53 -0400289 check_prerelease_version(rd.getVar('PV'), 'devtool add')
290
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600291 logger.info('Recipe %s has been automatically created; further editing may be required to make it fully functional' % recipefile)
292
293 finally:
294 tinfoil.shutdown()
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500295
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500296 return 0
297
298
299def _check_compatible_recipe(pn, d):
300 """Check if the recipe is supported by devtool"""
301 if pn == 'perf':
302 raise DevtoolError("The perf recipe does not actually check out "
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600303 "source and thus cannot be supported by this tool",
304 4)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500305
306 if pn in ['kernel-devsrc', 'package-index'] or pn.startswith('gcc-source'):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600307 raise DevtoolError("The %s recipe is not supported by this tool" % pn, 4)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500308
309 if bb.data.inherits_class('image', d):
310 raise DevtoolError("The %s recipe is an image, and therefore is not "
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600311 "supported by this tool" % pn, 4)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500312
313 if bb.data.inherits_class('populate_sdk', d):
314 raise DevtoolError("The %s recipe is an SDK, and therefore is not "
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600315 "supported by this tool" % pn, 4)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500316
317 if bb.data.inherits_class('packagegroup', d):
318 raise DevtoolError("The %s recipe is a packagegroup, and therefore is "
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600319 "not supported by this tool" % pn, 4)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500320
321 if bb.data.inherits_class('meta', d):
322 raise DevtoolError("The %s recipe is a meta-recipe, and therefore is "
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600323 "not supported by this tool" % pn, 4)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500324
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500325 if bb.data.inherits_class('externalsrc', d) and d.getVar('EXTERNALSRC'):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600326 # Not an incompatibility error per se, so we don't pass the error code
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500327 raise DevtoolError("externalsrc is currently enabled for the %s "
328 "recipe. This prevents the normal do_patch task "
329 "from working. You will need to disable this "
330 "first." % pn)
331
Brad Bishop316dfdd2018-06-25 12:45:53 -0400332def _dry_run_copy(src, dst, dry_run_outdir, base_outdir):
333 """Common function for copying a file to the dry run output directory"""
334 relpath = os.path.relpath(dst, base_outdir)
335 if relpath.startswith('..'):
336 raise Exception('Incorrect base path %s for path %s' % (base_outdir, dst))
337 dst = os.path.join(dry_run_outdir, relpath)
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500338 dst_d = os.path.dirname(dst)
339 if dst_d:
340 bb.utils.mkdirhier(dst_d)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400341 # Don't overwrite existing files, otherwise in the case of an upgrade
342 # the dry-run written out recipe will be overwritten with an unmodified
343 # version
344 if not os.path.exists(dst):
345 shutil.copy(src, dst)
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500346
Brad Bishop316dfdd2018-06-25 12:45:53 -0400347def _move_file(src, dst, dry_run_outdir=None, base_outdir=None):
348 """Move a file. Creates all the directory components of destination path."""
349 dry_run_suffix = ' (dry-run)' if dry_run_outdir else ''
350 logger.debug('Moving %s to %s%s' % (src, dst, dry_run_suffix))
351 if dry_run_outdir:
352 # We want to copy here, not move
353 _dry_run_copy(src, dst, dry_run_outdir, base_outdir)
354 else:
355 dst_d = os.path.dirname(dst)
356 if dst_d:
357 bb.utils.mkdirhier(dst_d)
358 shutil.move(src, dst)
359
360def _copy_file(src, dst, dry_run_outdir=None):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600361 """Copy a file. Creates all the directory components of destination path."""
Brad Bishop316dfdd2018-06-25 12:45:53 -0400362 dry_run_suffix = ' (dry-run)' if dry_run_outdir else ''
363 logger.debug('Copying %s to %s%s' % (src, dst, dry_run_suffix))
364 if dry_run_outdir:
365 _dry_run_copy(src, dst, dry_run_outdir, base_outdir)
366 else:
367 dst_d = os.path.dirname(dst)
368 if dst_d:
369 bb.utils.mkdirhier(dst_d)
370 shutil.copy(src, dst)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600371
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500372def _git_ls_tree(repodir, treeish='HEAD', recursive=False):
373 """List contents of a git treeish"""
374 import bb
375 cmd = ['git', 'ls-tree', '-z', treeish]
376 if recursive:
377 cmd.append('-r')
378 out, _ = bb.process.run(cmd, cwd=repodir)
379 ret = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500380 if out:
381 for line in out.split('\0'):
382 if line:
383 split = line.split(None, 4)
384 ret[split[3]] = split[0:3]
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500385 return ret
386
387def _git_exclude_path(srctree, path):
388 """Return pathspec (list of paths) that excludes certain path"""
389 # NOTE: "Filtering out" files/paths in this way is not entirely reliable -
390 # we don't catch files that are deleted, for example. A more reliable way
391 # to implement this would be to use "negative pathspecs" which were
392 # introduced in Git v1.9.0. Revisit this when/if the required Git version
393 # becomes greater than that.
394 path = os.path.normpath(path)
395 recurse = True if len(path.split(os.path.sep)) > 1 else False
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600396 git_files = list(_git_ls_tree(srctree, 'HEAD', recurse).keys())
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500397 if path in git_files:
398 git_files.remove(path)
399 return git_files
400 else:
401 return ['.']
402
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500403def _ls_tree(directory):
404 """Recursive listing of files in a directory"""
405 ret = []
406 for root, dirs, files in os.walk(directory):
407 ret.extend([os.path.relpath(os.path.join(root, fname), directory) for
408 fname in files])
409 return ret
410
411
412def extract(args, config, basepath, workspace):
413 """Entry point for the devtool 'extract' subcommand"""
414 import bb
415
Brad Bishop316dfdd2018-06-25 12:45:53 -0400416 tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500417 if not tinfoil:
418 # Error already shown
419 return 1
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600420 try:
421 rd = parse_recipe(config, tinfoil, args.recipename, True)
422 if not rd:
423 return 1
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500424
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600425 srctree = os.path.abspath(args.srctree)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400426 initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600427 logger.info('Source tree extracted to %s' % srctree)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500428
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600429 if initial_rev:
430 return 0
431 else:
432 return 1
433 finally:
434 tinfoil.shutdown()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500435
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500436def sync(args, config, basepath, workspace):
437 """Entry point for the devtool 'sync' subcommand"""
438 import bb
439
Brad Bishop316dfdd2018-06-25 12:45:53 -0400440 tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500441 if not tinfoil:
442 # Error already shown
443 return 1
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600444 try:
445 rd = parse_recipe(config, tinfoil, args.recipename, True)
446 if not rd:
447 return 1
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500448
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600449 srctree = os.path.abspath(args.srctree)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400450 initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, True, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=True)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600451 logger.info('Source tree %s synchronized' % srctree)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500452
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600453 if initial_rev:
454 return 0
455 else:
456 return 1
457 finally:
458 tinfoil.shutdown()
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500459
Brad Bishop96ff1982019-08-19 13:50:42 -0400460def symlink_oelocal_files_srctree(rd,srctree):
461 import oe.patch
462 if os.path.abspath(rd.getVar('S')) == os.path.abspath(rd.getVar('WORKDIR')):
463 # If recipe extracts to ${WORKDIR}, symlink the files into the srctree
464 # (otherwise the recipe won't build as expected)
465 local_files_dir = os.path.join(srctree, 'oe-local-files')
466 addfiles = []
467 for root, _, files in os.walk(local_files_dir):
468 relpth = os.path.relpath(root, local_files_dir)
469 if relpth != '.':
470 bb.utils.mkdirhier(os.path.join(srctree, relpth))
471 for fn in files:
472 if fn == '.gitignore':
473 continue
474 destpth = os.path.join(srctree, relpth, fn)
475 if os.path.exists(destpth):
476 os.unlink(destpth)
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600477 if relpth != '.':
478 back_relpth = os.path.relpath(local_files_dir, root)
479 os.symlink('%s/oe-local-files/%s/%s' % (back_relpth, relpth, fn), destpth)
480 else:
481 os.symlink('oe-local-files/%s' % fn, destpth)
Brad Bishop96ff1982019-08-19 13:50:42 -0400482 addfiles.append(os.path.join(relpth, fn))
483 if addfiles:
484 bb.process.run('git add %s' % ' '.join(addfiles), cwd=srctree)
Brad Bishop79641f22019-09-10 07:20:22 -0400485 useroptions = []
486 oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=rd)
487 bb.process.run('git %s commit -m "Committing local file symlinks\n\n%s"' % (' '.join(useroptions), oe.patch.GitApplyTree.ignore_commit_prefix), cwd=srctree)
Brad Bishop96ff1982019-08-19 13:50:42 -0400488
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500489
Brad Bishop316dfdd2018-06-25 12:45:53 -0400490def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, workspace, fixed_setup, d, tinfoil, no_overrides=False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500491 """Extract sources of a recipe"""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500492 import oe.recipeutils
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500493 import oe.patch
Brad Bishop96ff1982019-08-19 13:50:42 -0400494 import oe.path
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500495
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500496 pn = d.getVar('PN')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500497
498 _check_compatible_recipe(pn, d)
499
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500500 if sync:
501 if not os.path.exists(srctree):
502 raise DevtoolError("output path %s does not exist" % srctree)
503 else:
504 if os.path.exists(srctree):
505 if not os.path.isdir(srctree):
506 raise DevtoolError("output path %s exists and is not a directory" %
507 srctree)
508 elif os.listdir(srctree):
509 raise DevtoolError("output path %s already exists and is "
510 "non-empty" % srctree)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500511
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500512 if 'noexec' in (d.getVarFlags('do_unpack', False) or []):
513 raise DevtoolError("The %s recipe has do_unpack disabled, unable to "
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600514 "extract source" % pn, 4)
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500515
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500516 if not sync:
517 # Prepare for shutil.move later on
518 bb.utils.mkdirhier(srctree)
519 os.rmdir(srctree)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500520
Brad Bishop316dfdd2018-06-25 12:45:53 -0400521 extra_overrides = []
522 if not no_overrides:
523 history = d.varhistory.variable('SRC_URI')
524 for event in history:
525 if not 'flag' in event:
Patrick Williams213cb262021-08-07 19:21:33 -0500526 if event['op'].startswith((':append[', ':prepend[')):
Brad Bishop316dfdd2018-06-25 12:45:53 -0400527 extra_overrides.append(event['op'].split('[')[1].split(']')[0])
Andrew Geissler99467da2019-02-25 18:54:23 -0600528 # We want to remove duplicate overrides. If a recipe had multiple
529 # SRC_URI_override += values it would cause mulitple instances of
530 # overrides. This doesn't play nicely with things like creating a
531 # branch for every instance of DEVTOOL_EXTRA_OVERRIDES.
532 extra_overrides = list(set(extra_overrides))
Brad Bishop316dfdd2018-06-25 12:45:53 -0400533 if extra_overrides:
534 logger.info('SRC_URI contains some conditional appends/prepends - will create branches to represent these')
535
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500536 initial_rev = None
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500537
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500538 recipefile = d.getVar('FILE')
539 appendfile = recipe_to_append(recipefile, config)
540 is_kernel_yocto = bb.data.inherits_class('kernel-yocto', d)
541
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500542 # We need to redirect WORKDIR, STAMPS_DIR etc. under a temporary
543 # directory so that:
544 # (a) we pick up all files that get unpacked to the WORKDIR, and
545 # (b) we don't disturb the existing build
546 # However, with recipe-specific sysroots the sysroots for the recipe
547 # will be prepared under WORKDIR, and if we used the system temporary
548 # directory (i.e. usually /tmp) as used by mkdtemp by default, then
549 # our attempts to hardlink files into the recipe-specific sysroots
550 # will fail on systems where /tmp is a different filesystem, and it
551 # would have to fall back to copying the files which is a waste of
552 # time. Put the temp directory under the WORKDIR to prevent that from
553 # being a problem.
554 tempbasedir = d.getVar('WORKDIR')
555 bb.utils.mkdirhier(tempbasedir)
556 tempdir = tempfile.mkdtemp(prefix='devtooltmp-', dir=tempbasedir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500557 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500558 tinfoil.logger.setLevel(logging.WARNING)
559
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500560 # FIXME this results in a cache reload under control of tinfoil, which is fine
561 # except we don't get the knotty progress bar
562
563 if os.path.exists(appendfile):
564 appendbackup = os.path.join(tempdir, os.path.basename(appendfile) + '.bak')
565 shutil.copyfile(appendfile, appendbackup)
566 else:
567 appendbackup = None
568 bb.utils.mkdirhier(os.path.dirname(appendfile))
569 logger.debug('writing append file %s' % appendfile)
570 with open(appendfile, 'a') as f:
571 f.write('###--- _extract_source\n')
572 f.write('DEVTOOL_TEMPDIR = "%s"\n' % tempdir)
573 f.write('DEVTOOL_DEVBRANCH = "%s"\n' % devbranch)
574 if not is_kernel_yocto:
575 f.write('PATCHTOOL = "git"\n')
576 f.write('PATCH_COMMIT_FUNCTIONS = "1"\n')
Brad Bishop316dfdd2018-06-25 12:45:53 -0400577 if extra_overrides:
578 f.write('DEVTOOL_EXTRA_OVERRIDES = "%s"\n' % ':'.join(extra_overrides))
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500579 f.write('inherit devtool-source\n')
580 f.write('###--- _extract_source\n')
581
582 update_unlockedsigs(basepath, workspace, fixed_setup, [pn])
583
584 sstate_manifests = d.getVar('SSTATE_MANIFESTS')
585 bb.utils.mkdirhier(sstate_manifests)
586 preservestampfile = os.path.join(sstate_manifests, 'preserve-stamps')
587 with open(preservestampfile, 'w') as f:
588 f.write(d.getVar('STAMP'))
589 try:
Brad Bishop96ff1982019-08-19 13:50:42 -0400590 if is_kernel_yocto:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500591 # We need to generate the kernel config
592 task = 'do_configure'
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500593 else:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500594 task = 'do_patch'
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500595
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600596 if 'noexec' in (d.getVarFlags(task, False) or []) or 'task' not in (d.getVarFlags(task, False) or []):
597 logger.info('The %s recipe has %s disabled. Running only '
598 'do_configure task dependencies' % (pn, task))
599
600 if 'depends' in d.getVarFlags('do_configure', False):
601 pn = d.getVarFlags('do_configure', False)['depends']
602 pn = pn.replace('${PV}', d.getVar('PV'))
603 pn = pn.replace('${COMPILERDEP}', d.getVar('COMPILERDEP'))
604 task = None
605
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500606 # Run the fetch + unpack tasks
607 res = tinfoil.build_targets(pn,
608 task,
609 handle_events=True)
610 finally:
611 if os.path.exists(preservestampfile):
612 os.remove(preservestampfile)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500613
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500614 if not res:
615 raise DevtoolError('Extracting source for %s failed' % pn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500616
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600617 if not is_kernel_yocto and ('noexec' in (d.getVarFlags('do_patch', False) or []) or 'task' not in (d.getVarFlags('do_patch', False) or [])):
618 workshareddir = d.getVar('S')
619 if os.path.islink(srctree):
620 os.unlink(srctree)
621
622 os.symlink(workshareddir, srctree)
623
624 # The initial_rev file is created in devtool_post_unpack function that will not be executed if
625 # do_unpack/do_patch tasks are disabled so we have to directly say that source extraction was successful
626 return True, True
627
Brad Bishop316dfdd2018-06-25 12:45:53 -0400628 try:
629 with open(os.path.join(tempdir, 'initial_rev'), 'r') as f:
630 initial_rev = f.read()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500631
Brad Bishop316dfdd2018-06-25 12:45:53 -0400632 with open(os.path.join(tempdir, 'srcsubdir'), 'r') as f:
633 srcsubdir = f.read()
634 except FileNotFoundError as e:
635 raise DevtoolError('Something went wrong with source extraction - the devtool-source class was not active or did not function correctly:\n%s' % str(e))
636 srcsubdir_rel = os.path.relpath(srcsubdir, os.path.join(tempdir, 'workdir'))
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500637
Brad Bishop96ff1982019-08-19 13:50:42 -0400638 # Check if work-shared is empty, if yes
639 # find source and copy to work-shared
640 if is_kernel_yocto:
641 workshareddir = d.getVar('STAGING_KERNEL_DIR')
642 staging_kerVer = get_staging_kver(workshareddir)
643 kernelVersion = d.getVar('LINUX_VERSION')
644
645 # handle dangling symbolic link in work-shared:
646 if os.path.islink(workshareddir):
647 os.unlink(workshareddir)
648
649 if os.path.exists(workshareddir) and (not os.listdir(workshareddir) or kernelVersion != staging_kerVer):
650 shutil.rmtree(workshareddir)
651 oe.path.copyhardlinktree(srcsubdir,workshareddir)
652 elif not os.path.exists(workshareddir):
653 oe.path.copyhardlinktree(srcsubdir,workshareddir)
654
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500655 tempdir_localdir = os.path.join(tempdir, 'oe-local-files')
656 srctree_localdir = os.path.join(srctree, 'oe-local-files')
657
658 if sync:
659 bb.process.run('git fetch file://' + srcsubdir + ' ' + devbranch + ':' + devbranch, cwd=srctree)
660
661 # Move oe-local-files directory to srctree
662 # As the oe-local-files is not part of the constructed git tree,
663 # remove them directly during the synchrounizating might surprise
664 # the users. Instead, we move it to oe-local-files.bak and remind
665 # user in the log message.
666 if os.path.exists(srctree_localdir + '.bak'):
667 shutil.rmtree(srctree_localdir, srctree_localdir + '.bak')
668
669 if os.path.exists(srctree_localdir):
670 logger.info('Backing up current local file directory %s' % srctree_localdir)
671 shutil.move(srctree_localdir, srctree_localdir + '.bak')
672
673 if os.path.exists(tempdir_localdir):
674 logger.info('Syncing local source files to srctree...')
675 shutil.copytree(tempdir_localdir, srctree_localdir)
676 else:
677 # Move oe-local-files directory to srctree
678 if os.path.exists(tempdir_localdir):
679 logger.info('Adding local source files to srctree...')
680 shutil.move(tempdir_localdir, srcsubdir)
681
682 shutil.move(srcsubdir, srctree)
Brad Bishop96ff1982019-08-19 13:50:42 -0400683 symlink_oelocal_files_srctree(d,srctree)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500684
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500685 if is_kernel_yocto:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500686 logger.info('Copying kernel config to srctree')
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500687 shutil.copy2(os.path.join(tempdir, '.config'), srctree)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500688
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500689 finally:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500690 if appendbackup:
691 shutil.copyfile(appendbackup, appendfile)
692 elif os.path.exists(appendfile):
693 os.remove(appendfile)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500694 if keep_temp:
695 logger.info('Preserving temporary directory %s' % tempdir)
696 else:
697 shutil.rmtree(tempdir)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400698 return initial_rev, srcsubdir_rel
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500699
700def _add_md5(config, recipename, filename):
701 """Record checksum of a file (or recursively for a directory) to the md5-file of the workspace"""
702 import bb.utils
703
704 def addfile(fn):
705 md5 = bb.utils.md5_file(fn)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500706 with open(os.path.join(config.workspace_path, '.devtool_md5'), 'a+') as f:
707 md5_str = '%s|%s|%s\n' % (recipename, os.path.relpath(fn, config.workspace_path), md5)
708 f.seek(0, os.SEEK_SET)
709 if not md5_str in f.read():
710 f.write(md5_str)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500711
712 if os.path.isdir(filename):
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500713 for root, _, files in os.walk(filename):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500714 for f in files:
715 addfile(os.path.join(root, f))
716 else:
717 addfile(filename)
718
719def _check_preserve(config, recipename):
720 """Check if a file was manually changed and needs to be saved in 'attic'
721 directory"""
722 import bb.utils
723 origfile = os.path.join(config.workspace_path, '.devtool_md5')
724 newfile = os.path.join(config.workspace_path, '.devtool_md5_new')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500725 preservepath = os.path.join(config.workspace_path, 'attic', recipename)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500726 with open(origfile, 'r') as f:
727 with open(newfile, 'w') as tf:
728 for line in f.readlines():
729 splitline = line.rstrip().split('|')
730 if splitline[0] == recipename:
731 removefile = os.path.join(config.workspace_path, splitline[1])
732 try:
733 md5 = bb.utils.md5_file(removefile)
734 except IOError as err:
735 if err.errno == 2:
736 # File no longer exists, skip it
737 continue
738 else:
739 raise
740 if splitline[2] != md5:
741 bb.utils.mkdirhier(preservepath)
742 preservefile = os.path.basename(removefile)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800743 logger.warning('File %s modified since it was written, preserving in %s' % (preservefile, preservepath))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500744 shutil.move(removefile, os.path.join(preservepath, preservefile))
745 else:
746 os.remove(removefile)
747 else:
748 tf.write(line)
Andrew Geisslerc926e172021-05-07 16:11:35 -0500749 bb.utils.rename(newfile, origfile)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500750
Brad Bishop96ff1982019-08-19 13:50:42 -0400751def get_staging_kver(srcdir):
752 # Kernel version from work-shared
753 kerver = []
754 staging_kerVer=""
755 if os.path.exists(srcdir) and os.listdir(srcdir):
756 with open(os.path.join(srcdir,"Makefile")) as f:
757 version = [next(f) for x in range(5)][1:4]
758 for word in version:
759 kerver.append(word.split('= ')[1].split('\n')[0])
760 staging_kerVer = ".".join(kerver)
761 return staging_kerVer
762
763def get_staging_kbranch(srcdir):
764 staging_kbranch = ""
765 if os.path.exists(srcdir) and os.listdir(srcdir):
766 (branch, _) = bb.process.run('git branch | grep \* | cut -d \' \' -f2', cwd=srcdir)
767 staging_kbranch = "".join(branch.split('\n')[0])
768 return staging_kbranch
769
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500770def modify(args, config, basepath, workspace):
771 """Entry point for the devtool 'modify' subcommand"""
772 import bb
773 import oe.recipeutils
Brad Bishop316dfdd2018-06-25 12:45:53 -0400774 import oe.patch
Brad Bishop96ff1982019-08-19 13:50:42 -0400775 import oe.path
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500776
777 if args.recipename in workspace:
778 raise DevtoolError("recipe %s is already in your workspace" %
779 args.recipename)
780
Brad Bishop316dfdd2018-06-25 12:45:53 -0400781 tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600782 try:
783 rd = parse_recipe(config, tinfoil, args.recipename, True)
784 if not rd:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500785 return 1
786
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500787 pn = rd.getVar('PN')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600788 if pn != args.recipename:
789 logger.info('Mapping %s to %s' % (args.recipename, pn))
790 if pn in workspace:
791 raise DevtoolError("recipe %s is already in your workspace" %
792 pn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500793
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600794 if args.srctree:
795 srctree = os.path.abspath(args.srctree)
796 else:
797 srctree = get_default_srctree(config, pn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500798
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600799 if args.no_extract and not os.path.isdir(srctree):
800 raise DevtoolError("--no-extract specified and source path %s does "
801 "not exist or is not a directory" %
802 srctree)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600803
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500804 recipefile = rd.getVar('FILE')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600805 appendfile = recipe_to_append(recipefile, config, args.wildcard)
806 if os.path.exists(appendfile):
807 raise DevtoolError("Another variant of recipe %s is already in your "
808 "workspace (only one variant of a recipe can "
809 "currently be worked on at once)"
810 % pn)
811
812 _check_compatible_recipe(pn, rd)
813
814 initial_rev = None
815 commits = []
Brad Bishop316dfdd2018-06-25 12:45:53 -0400816 check_commits = False
Brad Bishop96ff1982019-08-19 13:50:42 -0400817
818 if bb.data.inherits_class('kernel-yocto', rd):
819 # Current set kernel version
820 kernelVersion = rd.getVar('LINUX_VERSION')
821 srcdir = rd.getVar('STAGING_KERNEL_DIR')
822 kbranch = rd.getVar('KBRANCH')
823
824 staging_kerVer = get_staging_kver(srcdir)
825 staging_kbranch = get_staging_kbranch(srcdir)
826 if (os.path.exists(srcdir) and os.listdir(srcdir)) and (kernelVersion in staging_kerVer and staging_kbranch == kbranch):
827 oe.path.copyhardlinktree(srcdir,srctree)
828 workdir = rd.getVar('WORKDIR')
829 srcsubdir = rd.getVar('S')
830 localfilesdir = os.path.join(srctree,'oe-local-files')
831 # Move local source files into separate subdir
832 recipe_patches = [os.path.basename(patch) for patch in oe.recipeutils.get_recipe_patches(rd)]
833 local_files = oe.recipeutils.get_recipe_local_files(rd)
834
835 for key in local_files.copy():
836 if key.endswith('scc'):
837 sccfile = open(local_files[key], 'r')
838 for l in sccfile:
839 line = l.split()
840 if line and line[0] in ('kconf', 'patch'):
841 cfg = os.path.join(os.path.dirname(local_files[key]), line[-1])
842 if not cfg in local_files.values():
843 local_files[line[-1]] = cfg
844 shutil.copy2(cfg, workdir)
845 sccfile.close()
846
847 # Ignore local files with subdir={BP}
848 srcabspath = os.path.abspath(srcsubdir)
849 local_files = [fname for fname in local_files if os.path.exists(os.path.join(workdir, fname)) and (srcabspath == workdir or not os.path.join(workdir, fname).startswith(srcabspath + os.sep))]
850 if local_files:
851 for fname in local_files:
852 _move_file(os.path.join(workdir, fname), os.path.join(srctree, 'oe-local-files', fname))
853 with open(os.path.join(srctree, 'oe-local-files', '.gitignore'), 'w') as f:
854 f.write('# Ignore local files, by default. Remove this file ''if you want to commit the directory to Git\n*\n')
855
856 symlink_oelocal_files_srctree(rd,srctree)
857
858 task = 'do_configure'
859 res = tinfoil.build_targets(pn, task, handle_events=True)
860
861 # Copy .config to workspace
862 kconfpath = rd.getVar('B')
863 logger.info('Copying kernel config to workspace')
864 shutil.copy2(os.path.join(kconfpath, '.config'),srctree)
865
866 # Set this to true, we still need to get initial_rev
867 # by parsing the git repo
868 args.no_extract = True
869
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600870 if not args.no_extract:
Brad Bishop316dfdd2018-06-25 12:45:53 -0400871 initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500872 if not initial_rev:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600873 return 1
874 logger.info('Source tree extracted to %s' % srctree)
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600875 if os.path.exists(os.path.join(srctree, '.git')):
876 # Get list of commits since this revision
877 (stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % initial_rev, cwd=srctree)
878 commits = stdout.split()
879 check_commits = True
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600880 else:
881 if os.path.exists(os.path.join(srctree, '.git')):
Andrew Geissler99467da2019-02-25 18:54:23 -0600882 # Check if it's a tree previously extracted by us. This is done
883 # by ensuring that devtool-base and args.branch (devtool) exist.
884 # The check_commits logic will cause an exception if either one
885 # of these doesn't exist
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600886 try:
887 (stdout, _) = bb.process.run('git branch --contains devtool-base', cwd=srctree)
Andrew Geissler99467da2019-02-25 18:54:23 -0600888 bb.process.run('git rev-parse %s' % args.branch, cwd=srctree)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600889 except bb.process.ExecutionError:
890 stdout = ''
Brad Bishop316dfdd2018-06-25 12:45:53 -0400891 if stdout:
892 check_commits = True
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600893 for line in stdout.splitlines():
894 if line.startswith('*'):
895 (stdout, _) = bb.process.run('git rev-parse devtool-base', cwd=srctree)
896 initial_rev = stdout.rstrip()
897 if not initial_rev:
898 # Otherwise, just grab the head revision
899 (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree)
900 initial_rev = stdout.rstrip()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500901
Brad Bishop316dfdd2018-06-25 12:45:53 -0400902 branch_patches = {}
903 if check_commits:
904 # Check if there are override branches
905 (stdout, _) = bb.process.run('git branch', cwd=srctree)
906 branches = []
907 for line in stdout.rstrip().splitlines():
908 branchname = line[2:].rstrip()
909 if branchname.startswith(override_branch_prefix):
910 branches.append(branchname)
911 if branches:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800912 logger.warning('SRC_URI is conditionally overridden in this recipe, thus several %s* branches have been created, one for each override that makes changes to SRC_URI. It is recommended that you make changes to the %s branch first, then checkout and rebase each %s* branch and update any unique patches there (duplicates on those branches will be ignored by devtool finish/update-recipe)' % (override_branch_prefix, args.branch, override_branch_prefix))
Brad Bishop316dfdd2018-06-25 12:45:53 -0400913 branches.insert(0, args.branch)
914 seen_patches = []
915 for branch in branches:
916 branch_patches[branch] = []
917 (stdout, _) = bb.process.run('git log devtool-base..%s' % branch, cwd=srctree)
918 for line in stdout.splitlines():
919 line = line.strip()
920 if line.startswith(oe.patch.GitApplyTree.patch_line_prefix):
921 origpatch = line[len(oe.patch.GitApplyTree.patch_line_prefix):].split(':', 1)[-1].strip()
922 if not origpatch in seen_patches:
923 seen_patches.append(origpatch)
924 branch_patches[branch].append(origpatch)
925
926 # Need to grab this here in case the source is within a subdirectory
927 srctreebase = srctree
928
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600929 # Check that recipe isn't using a shared workdir
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500930 s = os.path.abspath(rd.getVar('S'))
931 workdir = os.path.abspath(rd.getVar('WORKDIR'))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600932 if s.startswith(workdir) and s != workdir and os.path.dirname(s) != workdir:
933 # Handle if S is set to a subdirectory of the source
934 srcsubdir = os.path.relpath(s, workdir).split(os.sep, 1)[1]
935 srctree = os.path.join(srctree, srcsubdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500936
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600937 bb.utils.mkdirhier(os.path.dirname(appendfile))
938 with open(appendfile, 'w') as f:
Patrick Williams213cb262021-08-07 19:21:33 -0500939 f.write('FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600940 # Local files can be modified/tracked in separate subdir under srctree
941 # Mostly useful for packages with S != WORKDIR
Patrick Williams213cb262021-08-07 19:21:33 -0500942 f.write('FILESPATH:prepend := "%s:"\n' %
Brad Bishop316dfdd2018-06-25 12:45:53 -0400943 os.path.join(srctreebase, 'oe-local-files'))
944 f.write('# srctreebase: %s\n' % srctreebase)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500945
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600946 f.write('\ninherit externalsrc\n')
947 f.write('# NOTE: We use pn- overrides here to avoid affecting multiple variants in the case where the recipe uses BBCLASSEXTEND\n')
Patrick Williams213cb262021-08-07 19:21:33 -0500948 f.write('EXTERNALSRC:pn-%s = "%s"\n' % (pn, srctree))
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500949
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600950 b_is_s = use_external_build(args.same_dir, args.no_same_dir, rd)
951 if b_is_s:
Patrick Williams213cb262021-08-07 19:21:33 -0500952 f.write('EXTERNALSRC_BUILD:pn-%s = "%s"\n' % (pn, srctree))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500953
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600954 if bb.data.inherits_class('kernel', rd):
955 f.write('SRCTREECOVEREDTASKS = "do_validate_branches do_kernel_checkout '
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500956 'do_fetch do_unpack do_kernel_configcheck"\n')
Brad Bishop19323692019-04-05 15:28:33 -0400957 f.write('\ndo_patch[noexec] = "1"\n')
Patrick Williams213cb262021-08-07 19:21:33 -0500958 f.write('\ndo_configure:append() {\n'
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600959 ' cp ${B}/.config ${S}/.config.baseline\n'
960 ' ln -sfT ${B}/.config ${S}/.config.new\n'
961 '}\n')
Patrick Williams213cb262021-08-07 19:21:33 -0500962 f.write('\ndo_kernel_configme:prepend() {\n'
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500963 ' if [ -e ${S}/.config ]; then\n'
964 ' mv ${S}/.config ${S}/.config.old\n'
965 ' fi\n'
966 '}\n')
Brad Bishop96ff1982019-08-19 13:50:42 -0400967 if rd.getVarFlag('do_menuconfig','task'):
Patrick Williams213cb262021-08-07 19:21:33 -0500968 f.write('\ndo_configure:append() {\n'
Andrew Geissler82c905d2020-04-13 13:39:40 -0500969 ' if [ ! ${DEVTOOL_DISABLE_MENUCONFIG} ]; then\n'
970 ' cp ${B}/.config ${S}/.config.baseline\n'
971 ' ln -sfT ${B}/.config ${S}/.config.new\n'
972 ' fi\n'
Brad Bishop96ff1982019-08-19 13:50:42 -0400973 '}\n')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600974 if initial_rev:
975 f.write('\n# initial_rev: %s\n' % initial_rev)
976 for commit in commits:
977 f.write('# commit: %s\n' % commit)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400978 if branch_patches:
979 for branch in branch_patches:
980 if branch == args.branch:
981 continue
982 f.write('# patches_%s: %s\n' % (branch, ','.join(branch_patches[branch])))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500983
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500984 update_unlockedsigs(basepath, workspace, args.fixed_setup, [pn])
985
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600986 _add_md5(config, pn, appendfile)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500987
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600988 logger.info('Recipe %s now set up to build from %s' % (pn, srctree))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500989
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600990 finally:
991 tinfoil.shutdown()
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500992
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500993 return 0
994
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500995
996def rename(args, config, basepath, workspace):
997 """Entry point for the devtool 'rename' subcommand"""
998 import bb
999 import oe.recipeutils
1000
1001 check_workspace_recipe(workspace, args.recipename)
1002
1003 if not (args.newname or args.version):
1004 raise DevtoolError('You must specify a new name, a version with -V/--version, or both')
1005
1006 recipefile = workspace[args.recipename]['recipefile']
1007 if not recipefile:
1008 raise DevtoolError('devtool rename can only be used where the recipe file itself is in the workspace (e.g. after devtool add)')
1009
1010 if args.newname and args.newname != args.recipename:
1011 reason = oe.recipeutils.validate_pn(args.newname)
1012 if reason:
1013 raise DevtoolError(reason)
1014 newname = args.newname
1015 else:
1016 newname = args.recipename
1017
1018 append = workspace[args.recipename]['bbappend']
1019 appendfn = os.path.splitext(os.path.basename(append))[0]
1020 splitfn = appendfn.split('_')
1021 if len(splitfn) > 1:
1022 origfnver = appendfn.split('_')[1]
1023 else:
1024 origfnver = ''
1025
1026 recipefilemd5 = None
1027 tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
1028 try:
1029 rd = parse_recipe(config, tinfoil, args.recipename, True)
1030 if not rd:
1031 return 1
1032
1033 bp = rd.getVar('BP')
1034 bpn = rd.getVar('BPN')
1035 if newname != args.recipename:
1036 localdata = rd.createCopy()
1037 localdata.setVar('PN', newname)
1038 newbpn = localdata.getVar('BPN')
1039 else:
1040 newbpn = bpn
1041 s = rd.getVar('S', False)
1042 src_uri = rd.getVar('SRC_URI', False)
1043 pv = rd.getVar('PV')
1044
1045 # Correct variable values that refer to the upstream source - these
1046 # values must stay the same, so if the name/version are changing then
1047 # we need to fix them up
1048 new_s = s
1049 new_src_uri = src_uri
1050 if newbpn != bpn:
1051 # ${PN} here is technically almost always incorrect, but people do use it
1052 new_s = new_s.replace('${BPN}', bpn)
1053 new_s = new_s.replace('${PN}', bpn)
1054 new_s = new_s.replace('${BP}', '%s-${PV}' % bpn)
1055 new_src_uri = new_src_uri.replace('${BPN}', bpn)
1056 new_src_uri = new_src_uri.replace('${PN}', bpn)
1057 new_src_uri = new_src_uri.replace('${BP}', '%s-${PV}' % bpn)
1058 if args.version and origfnver == pv:
1059 new_s = new_s.replace('${PV}', pv)
1060 new_s = new_s.replace('${BP}', '${BPN}-%s' % pv)
1061 new_src_uri = new_src_uri.replace('${PV}', pv)
1062 new_src_uri = new_src_uri.replace('${BP}', '${BPN}-%s' % pv)
1063 patchfields = {}
1064 if new_s != s:
1065 patchfields['S'] = new_s
1066 if new_src_uri != src_uri:
1067 patchfields['SRC_URI'] = new_src_uri
1068 if patchfields:
1069 recipefilemd5 = bb.utils.md5_file(recipefile)
1070 oe.recipeutils.patch_recipe(rd, recipefile, patchfields)
1071 newrecipefilemd5 = bb.utils.md5_file(recipefile)
1072 finally:
1073 tinfoil.shutdown()
1074
1075 if args.version:
1076 newver = args.version
1077 else:
1078 newver = origfnver
1079
1080 if newver:
1081 newappend = '%s_%s.bbappend' % (newname, newver)
1082 newfile = '%s_%s.bb' % (newname, newver)
1083 else:
1084 newappend = '%s.bbappend' % newname
1085 newfile = '%s.bb' % newname
1086
1087 oldrecipedir = os.path.dirname(recipefile)
1088 newrecipedir = os.path.join(config.workspace_path, 'recipes', newname)
1089 if oldrecipedir != newrecipedir:
1090 bb.utils.mkdirhier(newrecipedir)
1091
1092 newappend = os.path.join(os.path.dirname(append), newappend)
1093 newfile = os.path.join(newrecipedir, newfile)
1094
1095 # Rename bbappend
1096 logger.info('Renaming %s to %s' % (append, newappend))
Andrew Geisslerc926e172021-05-07 16:11:35 -05001097 bb.utils.rename(append, newappend)
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001098 # Rename recipe file
1099 logger.info('Renaming %s to %s' % (recipefile, newfile))
Andrew Geisslerc926e172021-05-07 16:11:35 -05001100 bb.utils.rename(recipefile, newfile)
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001101
1102 # Rename source tree if it's the default path
1103 appendmd5 = None
1104 if not args.no_srctree:
1105 srctree = workspace[args.recipename]['srctree']
1106 if os.path.abspath(srctree) == os.path.join(config.workspace_path, 'sources', args.recipename):
1107 newsrctree = os.path.join(config.workspace_path, 'sources', newname)
1108 logger.info('Renaming %s to %s' % (srctree, newsrctree))
1109 shutil.move(srctree, newsrctree)
1110 # Correct any references (basically EXTERNALSRC*) in the .bbappend
1111 appendmd5 = bb.utils.md5_file(newappend)
1112 appendlines = []
1113 with open(newappend, 'r') as f:
1114 for line in f:
1115 appendlines.append(line)
1116 with open(newappend, 'w') as f:
1117 for line in appendlines:
1118 if srctree in line:
1119 line = line.replace(srctree, newsrctree)
1120 f.write(line)
1121 newappendmd5 = bb.utils.md5_file(newappend)
1122
1123 bpndir = None
1124 newbpndir = None
1125 if newbpn != bpn:
1126 bpndir = os.path.join(oldrecipedir, bpn)
1127 if os.path.exists(bpndir):
1128 newbpndir = os.path.join(newrecipedir, newbpn)
1129 logger.info('Renaming %s to %s' % (bpndir, newbpndir))
1130 shutil.move(bpndir, newbpndir)
1131
1132 bpdir = None
1133 newbpdir = None
1134 if newver != origfnver or newbpn != bpn:
1135 bpdir = os.path.join(oldrecipedir, bp)
1136 if os.path.exists(bpdir):
1137 newbpdir = os.path.join(newrecipedir, '%s-%s' % (newbpn, newver))
1138 logger.info('Renaming %s to %s' % (bpdir, newbpdir))
1139 shutil.move(bpdir, newbpdir)
1140
1141 if oldrecipedir != newrecipedir:
1142 # Move any stray files and delete the old recipe directory
1143 for entry in os.listdir(oldrecipedir):
1144 oldpath = os.path.join(oldrecipedir, entry)
1145 newpath = os.path.join(newrecipedir, entry)
1146 logger.info('Renaming %s to %s' % (oldpath, newpath))
1147 shutil.move(oldpath, newpath)
1148 os.rmdir(oldrecipedir)
1149
1150 # Now take care of entries in .devtool_md5
1151 md5entries = []
1152 with open(os.path.join(config.workspace_path, '.devtool_md5'), 'r') as f:
1153 for line in f:
1154 md5entries.append(line)
1155
1156 if bpndir and newbpndir:
1157 relbpndir = os.path.relpath(bpndir, config.workspace_path) + '/'
1158 else:
1159 relbpndir = None
1160 if bpdir and newbpdir:
1161 relbpdir = os.path.relpath(bpdir, config.workspace_path) + '/'
1162 else:
1163 relbpdir = None
1164
1165 with open(os.path.join(config.workspace_path, '.devtool_md5'), 'w') as f:
1166 for entry in md5entries:
1167 splitentry = entry.rstrip().split('|')
1168 if len(splitentry) > 2:
1169 if splitentry[0] == args.recipename:
1170 splitentry[0] = newname
1171 if splitentry[1] == os.path.relpath(append, config.workspace_path):
1172 splitentry[1] = os.path.relpath(newappend, config.workspace_path)
1173 if appendmd5 and splitentry[2] == appendmd5:
1174 splitentry[2] = newappendmd5
1175 elif splitentry[1] == os.path.relpath(recipefile, config.workspace_path):
1176 splitentry[1] = os.path.relpath(newfile, config.workspace_path)
1177 if recipefilemd5 and splitentry[2] == recipefilemd5:
1178 splitentry[2] = newrecipefilemd5
1179 elif relbpndir and splitentry[1].startswith(relbpndir):
1180 splitentry[1] = os.path.relpath(os.path.join(newbpndir, splitentry[1][len(relbpndir):]), config.workspace_path)
1181 elif relbpdir and splitentry[1].startswith(relbpdir):
1182 splitentry[1] = os.path.relpath(os.path.join(newbpdir, splitentry[1][len(relbpdir):]), config.workspace_path)
1183 entry = '|'.join(splitentry) + '\n'
1184 f.write(entry)
1185 return 0
1186
1187
Brad Bishop316dfdd2018-06-25 12:45:53 -04001188def _get_patchset_revs(srctree, recipe_path, initial_rev=None, force_patch_refresh=False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001189 """Get initial and update rev of a recipe. These are the start point of the
1190 whole patchset and start point for the patches to be re-generated/updated.
1191 """
1192 import bb
1193
Brad Bishop316dfdd2018-06-25 12:45:53 -04001194 # Get current branch
1195 stdout, _ = bb.process.run('git rev-parse --abbrev-ref HEAD',
1196 cwd=srctree)
1197 branchname = stdout.rstrip()
1198
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001199 # Parse initial rev from recipe if not specified
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001200 commits = []
Brad Bishop316dfdd2018-06-25 12:45:53 -04001201 patches = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001202 with open(recipe_path, 'r') as f:
1203 for line in f:
1204 if line.startswith('# initial_rev:'):
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001205 if not initial_rev:
1206 initial_rev = line.split(':')[-1].strip()
Brad Bishop316dfdd2018-06-25 12:45:53 -04001207 elif line.startswith('# commit:') and not force_patch_refresh:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001208 commits.append(line.split(':')[-1].strip())
Brad Bishop316dfdd2018-06-25 12:45:53 -04001209 elif line.startswith('# patches_%s:' % branchname):
1210 patches = line.split(':')[-1].strip().split(',')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001211
1212 update_rev = initial_rev
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001213 changed_revs = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001214 if initial_rev:
1215 # Find first actually changed revision
1216 stdout, _ = bb.process.run('git rev-list --reverse %s..HEAD' %
1217 initial_rev, cwd=srctree)
1218 newcommits = stdout.split()
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001219 for i in range(min(len(commits), len(newcommits))):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001220 if newcommits[i] == commits[i]:
1221 update_rev = commits[i]
1222
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001223 try:
1224 stdout, _ = bb.process.run('git cherry devtool-patched',
1225 cwd=srctree)
1226 except bb.process.ExecutionError as err:
1227 stdout = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001228
Brad Bishop316dfdd2018-06-25 12:45:53 -04001229 if stdout is not None and not force_patch_refresh:
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001230 changed_revs = []
1231 for line in stdout.splitlines():
1232 if line.startswith('+ '):
1233 rev = line.split()[1]
1234 if rev in newcommits:
1235 changed_revs.append(rev)
1236
Brad Bishop316dfdd2018-06-25 12:45:53 -04001237 return initial_rev, update_rev, changed_revs, patches
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001238
1239def _remove_file_entries(srcuri, filelist):
1240 """Remove file:// entries from SRC_URI"""
1241 remaining = filelist[:]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001242 entries = []
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001243 for fname in filelist:
1244 basename = os.path.basename(fname)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001245 for i in range(len(srcuri)):
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001246 if (srcuri[i].startswith('file://') and
1247 os.path.basename(srcuri[i].split(';')[0]) == basename):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001248 entries.append(srcuri[i])
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001249 remaining.remove(fname)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001250 srcuri.pop(i)
1251 break
1252 return entries, remaining
1253
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001254def _replace_srcuri_entry(srcuri, filename, newentry):
1255 """Replace entry corresponding to specified file with a new entry"""
1256 basename = os.path.basename(filename)
1257 for i in range(len(srcuri)):
1258 if os.path.basename(srcuri[i].split(';')[0]) == basename:
1259 srcuri.pop(i)
1260 srcuri.insert(i, newentry)
1261 break
1262
Brad Bishop316dfdd2018-06-25 12:45:53 -04001263def _remove_source_files(append, files, destpath, no_report_remove=False, dry_run=False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001264 """Unlink existing patch files"""
Brad Bishop316dfdd2018-06-25 12:45:53 -04001265
1266 dry_run_suffix = ' (dry-run)' if dry_run else ''
1267
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001268 for path in files:
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001269 if append:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001270 if not destpath:
1271 raise Exception('destpath should be set here')
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001272 path = os.path.join(destpath, os.path.basename(path))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001273
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001274 if os.path.exists(path):
Brad Bishop316dfdd2018-06-25 12:45:53 -04001275 if not no_report_remove:
1276 logger.info('Removing file %s%s' % (path, dry_run_suffix))
1277 if not dry_run:
1278 # FIXME "git rm" here would be nice if the file in question is
1279 # tracked
1280 # FIXME there's a chance that this file is referred to by
1281 # another recipe, in which case deleting wouldn't be the
1282 # right thing to do
1283 os.remove(path)
1284 # Remove directory if empty
1285 try:
1286 os.rmdir(os.path.dirname(path))
1287 except OSError as ose:
1288 if ose.errno != errno.ENOTEMPTY:
1289 raise
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001290
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001291
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001292def _export_patches(srctree, rd, start_rev, destdir, changed_revs=None):
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001293 """Export patches from srctree to given location.
1294 Returns three-tuple of dicts:
1295 1. updated - patches that already exist in SRCURI
1296 2. added - new patches that don't exist in SRCURI
1297 3 removed - patches that exist in SRCURI but not in exported patches
1298 In each dict the key is the 'basepath' of the URI and value is the
1299 absolute path to the existing file in recipe space (if any).
1300 """
1301 import oe.recipeutils
1302 from oe.patch import GitApplyTree
1303 updated = OrderedDict()
1304 added = OrderedDict()
1305 seqpatch_re = re.compile('^([0-9]{4}-)?(.+)')
1306
1307 existing_patches = dict((os.path.basename(path), path) for path in
1308 oe.recipeutils.get_recipe_patches(rd))
Brad Bishop316dfdd2018-06-25 12:45:53 -04001309 logger.debug('Existing patches: %s' % existing_patches)
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001310
1311 # Generate patches from Git, exclude local files directory
1312 patch_pathspec = _git_exclude_path(srctree, 'oe-local-files')
1313 GitApplyTree.extractPatches(srctree, start_rev, destdir, patch_pathspec)
1314
1315 new_patches = sorted(os.listdir(destdir))
1316 for new_patch in new_patches:
1317 # Strip numbering from patch names. If it's a git sequence named patch,
1318 # the numbers might not match up since we are starting from a different
1319 # revision This does assume that people are using unique shortlog
1320 # values, but they ought to be anyway...
1321 new_basename = seqpatch_re.match(new_patch).group(2)
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001322 match_name = None
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001323 for old_patch in existing_patches:
1324 old_basename = seqpatch_re.match(old_patch).group(2)
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001325 old_basename_splitext = os.path.splitext(old_basename)
1326 if old_basename.endswith(('.gz', '.bz2', '.Z')) and old_basename_splitext[0] == new_basename:
1327 old_patch_noext = os.path.splitext(old_patch)[0]
1328 match_name = old_patch_noext
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001329 break
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001330 elif new_basename == old_basename:
1331 match_name = old_patch
1332 break
1333 if match_name:
1334 # Rename patch files
1335 if new_patch != match_name:
Andrew Geisslerc926e172021-05-07 16:11:35 -05001336 bb.utils.rename(os.path.join(destdir, new_patch),
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001337 os.path.join(destdir, match_name))
1338 # Need to pop it off the list now before checking changed_revs
1339 oldpath = existing_patches.pop(old_patch)
1340 if changed_revs is not None:
1341 # Avoid updating patches that have not actually changed
1342 with open(os.path.join(destdir, match_name), 'r') as f:
1343 firstlineitems = f.readline().split()
1344 # Looking for "From <hash>" line
1345 if len(firstlineitems) > 1 and len(firstlineitems[1]) == 40:
1346 if not firstlineitems[1] in changed_revs:
1347 continue
1348 # Recompress if necessary
1349 if oldpath.endswith(('.gz', '.Z')):
1350 bb.process.run(['gzip', match_name], cwd=destdir)
1351 if oldpath.endswith('.gz'):
1352 match_name += '.gz'
1353 else:
1354 match_name += '.Z'
1355 elif oldpath.endswith('.bz2'):
1356 bb.process.run(['bzip2', match_name], cwd=destdir)
1357 match_name += '.bz2'
1358 updated[match_name] = oldpath
1359 else:
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001360 added[new_patch] = None
1361 return (updated, added, existing_patches)
1362
1363
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001364def _create_kconfig_diff(srctree, rd, outfile):
1365 """Create a kconfig fragment"""
1366 # Only update config fragment if both config files exist
1367 orig_config = os.path.join(srctree, '.config.baseline')
1368 new_config = os.path.join(srctree, '.config.new')
1369 if os.path.exists(orig_config) and os.path.exists(new_config):
1370 cmd = ['diff', '--new-line-format=%L', '--old-line-format=',
1371 '--unchanged-line-format=', orig_config, new_config]
1372 pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE,
1373 stderr=subprocess.PIPE)
1374 stdout, stderr = pipe.communicate()
1375 if pipe.returncode == 1:
1376 logger.info("Updating config fragment %s" % outfile)
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001377 with open(outfile, 'wb') as fobj:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001378 fobj.write(stdout)
1379 elif pipe.returncode == 0:
1380 logger.info("Would remove config fragment %s" % outfile)
1381 if os.path.exists(outfile):
1382 # Remove fragment file in case of empty diff
1383 logger.info("Removing config fragment %s" % outfile)
1384 os.unlink(outfile)
1385 else:
1386 raise bb.process.ExecutionError(cmd, pipe.returncode, stdout, stderr)
1387 return True
1388 return False
1389
1390
Brad Bishop316dfdd2018-06-25 12:45:53 -04001391def _export_local_files(srctree, rd, destdir, srctreebase):
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001392 """Copy local files from srctree to given location.
1393 Returns three-tuple of dicts:
1394 1. updated - files that already exist in SRCURI
1395 2. added - new files files that don't exist in SRCURI
1396 3 removed - files that exist in SRCURI but not in exported files
1397 In each dict the key is the 'basepath' of the URI and value is the
1398 absolute path to the existing file in recipe space (if any).
1399 """
1400 import oe.recipeutils
1401
1402 # Find out local files (SRC_URI files that exist in the "recipe space").
1403 # Local files that reside in srctree are not included in patch generation.
1404 # Instead they are directly copied over the original source files (in
1405 # recipe space).
1406 existing_files = oe.recipeutils.get_recipe_local_files(rd)
1407 new_set = None
1408 updated = OrderedDict()
1409 added = OrderedDict()
1410 removed = OrderedDict()
Brad Bishop316dfdd2018-06-25 12:45:53 -04001411 local_files_dir = os.path.join(srctreebase, 'oe-local-files')
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001412 git_files = _git_ls_tree(srctree)
1413 if 'oe-local-files' in git_files:
1414 # If tracked by Git, take the files from srctree HEAD. First get
1415 # the tree object of the directory
1416 tmp_index = os.path.join(srctree, '.git', 'index.tmp.devtool')
1417 tree = git_files['oe-local-files'][2]
1418 bb.process.run(['git', 'checkout', tree, '--', '.'], cwd=srctree,
1419 env=dict(os.environ, GIT_WORK_TREE=destdir,
1420 GIT_INDEX_FILE=tmp_index))
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001421 new_set = list(_git_ls_tree(srctree, tree, True).keys())
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001422 elif os.path.isdir(local_files_dir):
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001423 # If not tracked by Git, just copy from working copy
Brad Bishop316dfdd2018-06-25 12:45:53 -04001424 new_set = _ls_tree(local_files_dir)
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001425 bb.process.run(['cp', '-ax',
Brad Bishop316dfdd2018-06-25 12:45:53 -04001426 os.path.join(local_files_dir, '.'), destdir])
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001427 else:
1428 new_set = []
1429
1430 # Special handling for kernel config
1431 if bb.data.inherits_class('kernel-yocto', rd):
1432 fragment_fn = 'devtool-fragment.cfg'
1433 fragment_path = os.path.join(destdir, fragment_fn)
1434 if _create_kconfig_diff(srctree, rd, fragment_path):
1435 if os.path.exists(fragment_path):
1436 if fragment_fn not in new_set:
1437 new_set.append(fragment_fn)
1438 # Copy fragment to local-files
1439 if os.path.isdir(local_files_dir):
1440 shutil.copy2(fragment_path, local_files_dir)
1441 else:
1442 if fragment_fn in new_set:
1443 new_set.remove(fragment_fn)
1444 # Remove fragment from local-files
1445 if os.path.exists(os.path.join(local_files_dir, fragment_fn)):
1446 os.unlink(os.path.join(local_files_dir, fragment_fn))
1447
Brad Bishopd89cb5f2019-04-10 09:02:41 -04001448 # Special handling for cml1, ccmake, etc bbclasses that generated
1449 # configuration fragment files that are consumed as source files
1450 for frag_class, frag_name in [("cml1", "fragment.cfg"), ("ccmake", "site-file.cmake")]:
1451 if bb.data.inherits_class(frag_class, rd):
1452 srcpath = os.path.join(rd.getVar('WORKDIR'), frag_name)
1453 if os.path.exists(srcpath):
1454 if frag_name not in new_set:
1455 new_set.append(frag_name)
1456 # copy fragment into destdir
1457 shutil.copy2(srcpath, destdir)
1458 # copy fragment into local files if exists
1459 if os.path.isdir(local_files_dir):
1460 shutil.copy2(srcpath, local_files_dir)
1461
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001462 if new_set is not None:
1463 for fname in new_set:
1464 if fname in existing_files:
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001465 origpath = existing_files.pop(fname)
1466 workpath = os.path.join(local_files_dir, fname)
1467 if not filecmp.cmp(origpath, workpath):
1468 updated[fname] = origpath
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001469 elif fname != '.gitignore':
1470 added[fname] = None
1471
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001472 workdir = rd.getVar('WORKDIR')
1473 s = rd.getVar('S')
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001474 if not s.endswith(os.sep):
1475 s += os.sep
1476
1477 if workdir != s:
1478 # Handle files where subdir= was specified
1479 for fname in list(existing_files.keys()):
1480 # FIXME handle both subdir starting with BP and not?
1481 fworkpath = os.path.join(workdir, fname)
1482 if fworkpath.startswith(s):
1483 fpath = os.path.join(srctree, os.path.relpath(fworkpath, s))
1484 if os.path.exists(fpath):
1485 origpath = existing_files.pop(fname)
1486 if not filecmp.cmp(origpath, fpath):
1487 updated[fpath] = origpath
1488
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001489 removed = existing_files
1490 return (updated, added, removed)
1491
1492
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001493def _determine_files_dir(rd):
1494 """Determine the appropriate files directory for a recipe"""
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001495 recipedir = rd.getVar('FILE_DIRNAME')
1496 for entry in rd.getVar('FILESPATH').split(':'):
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001497 relpth = os.path.relpath(entry, recipedir)
1498 if not os.sep in relpth:
1499 # One (or zero) levels below only, so we don't put anything in machine-specific directories
1500 if os.path.isdir(entry):
1501 return entry
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001502 return os.path.join(recipedir, rd.getVar('BPN'))
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001503
1504
Brad Bishop316dfdd2018-06-25 12:45:53 -04001505def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001506 """Implement the 'srcrev' mode of update-recipe"""
1507 import bb
1508 import oe.recipeutils
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001509
Brad Bishop316dfdd2018-06-25 12:45:53 -04001510 dry_run_suffix = ' (dry-run)' if dry_run_outdir else ''
1511
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001512 recipefile = rd.getVar('FILE')
Brad Bishop316dfdd2018-06-25 12:45:53 -04001513 recipedir = os.path.basename(recipefile)
1514 logger.info('Updating SRCREV in recipe %s%s' % (recipedir, dry_run_suffix))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001515
1516 # Get HEAD revision
1517 try:
1518 stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree)
1519 except bb.process.ExecutionError as err:
1520 raise DevtoolError('Failed to get HEAD revision in %s: %s' %
1521 (srctree, err))
1522 srcrev = stdout.strip()
1523 if len(srcrev) != 40:
1524 raise DevtoolError('Invalid hash returned by git: %s' % stdout)
1525
1526 destpath = None
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001527 remove_files = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001528 patchfields = {}
1529 patchfields['SRCREV'] = srcrev
1530 orig_src_uri = rd.getVar('SRC_URI', False) or ''
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001531 srcuri = orig_src_uri.split()
1532 tempdir = tempfile.mkdtemp(prefix='devtool')
1533 update_srcuri = False
Brad Bishop316dfdd2018-06-25 12:45:53 -04001534 appendfile = None
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001535 try:
1536 local_files_dir = tempfile.mkdtemp(dir=tempdir)
Brad Bishop316dfdd2018-06-25 12:45:53 -04001537 srctreebase = workspace[recipename]['srctreebase']
1538 upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001539 if not no_remove:
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001540 # Find list of existing patches in recipe file
1541 patches_dir = tempfile.mkdtemp(dir=tempdir)
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001542 old_srcrev = rd.getVar('SRCREV') or ''
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001543 upd_p, new_p, del_p = _export_patches(srctree, rd, old_srcrev,
1544 patches_dir)
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001545 logger.debug('Patches: update %s, new %s, delete %s' % (dict(upd_p), dict(new_p), dict(del_p)))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001546
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001547 # Remove deleted local files and "overlapping" patches
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001548 remove_files = list(del_f.values()) + list(upd_p.values()) + list(del_p.values())
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001549 if remove_files:
1550 removedentries = _remove_file_entries(srcuri, remove_files)[0]
1551 update_srcuri = True
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001552
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001553 if appendlayerdir:
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001554 files = dict((os.path.join(local_files_dir, key), val) for
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001555 key, val in list(upd_f.items()) + list(new_f.items()))
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001556 removevalues = {}
1557 if update_srcuri:
1558 removevalues = {'SRC_URI': removedentries}
1559 patchfields['SRC_URI'] = '\\\n '.join(srcuri)
Brad Bishop316dfdd2018-06-25 12:45:53 -04001560 if dry_run_outdir:
1561 logger.info('Creating bbappend (dry-run)')
1562 else:
1563 appendfile, destpath = oe.recipeutils.bbappend_recipe(
1564 rd, appendlayerdir, files, wildcardver=wildcard_version,
1565 extralines=patchfields, removevalues=removevalues,
1566 redirect_output=dry_run_outdir)
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001567 else:
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001568 files_dir = _determine_files_dir(rd)
1569 for basepath, path in upd_f.items():
Brad Bishop316dfdd2018-06-25 12:45:53 -04001570 logger.info('Updating file %s%s' % (basepath, dry_run_suffix))
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001571 if os.path.isabs(basepath):
1572 # Original file (probably with subdir pointing inside source tree)
1573 # so we do not want to move it, just copy
Brad Bishop316dfdd2018-06-25 12:45:53 -04001574 _copy_file(basepath, path, dry_run_outdir=dry_run_outdir, base_outdir=recipedir)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001575 else:
Brad Bishop316dfdd2018-06-25 12:45:53 -04001576 _move_file(os.path.join(local_files_dir, basepath), path,
1577 dry_run_outdir=dry_run_outdir, base_outdir=recipedir)
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001578 update_srcuri= True
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001579 for basepath, path in new_f.items():
Brad Bishop316dfdd2018-06-25 12:45:53 -04001580 logger.info('Adding new file %s%s' % (basepath, dry_run_suffix))
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001581 _move_file(os.path.join(local_files_dir, basepath),
Brad Bishop316dfdd2018-06-25 12:45:53 -04001582 os.path.join(files_dir, basepath),
1583 dry_run_outdir=dry_run_outdir,
1584 base_outdir=recipedir)
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001585 srcuri.append('file://%s' % basepath)
1586 update_srcuri = True
1587 if update_srcuri:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001588 patchfields['SRC_URI'] = ' '.join(srcuri)
Brad Bishop316dfdd2018-06-25 12:45:53 -04001589 ret = oe.recipeutils.patch_recipe(rd, recipefile, patchfields, redirect_output=dry_run_outdir)
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001590 finally:
1591 shutil.rmtree(tempdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001592 if not 'git://' in orig_src_uri:
1593 logger.info('You will need to update SRC_URI within the recipe to '
1594 'point to a git repository where you have pushed your '
1595 'changes')
1596
Brad Bishop316dfdd2018-06-25 12:45:53 -04001597 _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove, dry_run=dry_run_outdir)
1598 return True, appendfile, remove_files
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001599
Brad Bishop316dfdd2018-06-25 12:45:53 -04001600def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev, dry_run_outdir=None, force_patch_refresh=False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001601 """Implement the 'patch' mode of update-recipe"""
1602 import bb
1603 import oe.recipeutils
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001604
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001605 recipefile = rd.getVar('FILE')
Brad Bishop316dfdd2018-06-25 12:45:53 -04001606 recipedir = os.path.dirname(recipefile)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001607 append = workspace[recipename]['bbappend']
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001608 if not os.path.exists(append):
1609 raise DevtoolError('unable to find workspace bbappend for recipe %s' %
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001610 recipename)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001611
Brad Bishop316dfdd2018-06-25 12:45:53 -04001612 initial_rev, update_rev, changed_revs, filter_patches = _get_patchset_revs(srctree, append, initial_rev, force_patch_refresh)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001613 if not initial_rev:
1614 raise DevtoolError('Unable to find initial revision - please specify '
1615 'it with --initial-rev')
1616
Brad Bishop316dfdd2018-06-25 12:45:53 -04001617 appendfile = None
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001618 dl_dir = rd.getVar('DL_DIR')
1619 if not dl_dir.endswith('/'):
1620 dl_dir += '/'
1621
Brad Bishop316dfdd2018-06-25 12:45:53 -04001622 dry_run_suffix = ' (dry-run)' if dry_run_outdir else ''
1623
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001624 tempdir = tempfile.mkdtemp(prefix='devtool')
1625 try:
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001626 local_files_dir = tempfile.mkdtemp(dir=tempdir)
Brad Bishop316dfdd2018-06-25 12:45:53 -04001627 if filter_patches:
1628 upd_f = {}
1629 new_f = {}
1630 del_f = {}
1631 else:
1632 srctreebase = workspace[recipename]['srctreebase']
1633 upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001634
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001635 remove_files = []
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001636 if not no_remove:
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001637 # Get all patches from source tree and check if any should be removed
1638 all_patches_dir = tempfile.mkdtemp(dir=tempdir)
Brad Bishop316dfdd2018-06-25 12:45:53 -04001639 _, _, del_p = _export_patches(srctree, rd, initial_rev,
1640 all_patches_dir)
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001641 # Remove deleted local files and patches
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001642 remove_files = list(del_f.values()) + list(del_p.values())
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001643
1644 # Get updated patches from source tree
1645 patches_dir = tempfile.mkdtemp(dir=tempdir)
Brad Bishop316dfdd2018-06-25 12:45:53 -04001646 upd_p, new_p, _ = _export_patches(srctree, rd, update_rev,
1647 patches_dir, changed_revs)
1648 logger.debug('Pre-filtering: update: %s, new: %s' % (dict(upd_p), dict(new_p)))
1649 if filter_patches:
Brad Bishop00e122a2019-10-05 11:10:57 -04001650 new_p = OrderedDict()
1651 upd_p = OrderedDict((k,v) for k,v in upd_p.items() if k in filter_patches)
Brad Bishop316dfdd2018-06-25 12:45:53 -04001652 remove_files = [f for f in remove_files if f in filter_patches]
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001653 updatefiles = False
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001654 updaterecipe = False
1655 destpath = None
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001656 srcuri = (rd.getVar('SRC_URI', False) or '').split()
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001657 if appendlayerdir:
Brad Bishop00e122a2019-10-05 11:10:57 -04001658 files = OrderedDict((os.path.join(local_files_dir, key), val) for
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001659 key, val in list(upd_f.items()) + list(new_f.items()))
Brad Bishop00e122a2019-10-05 11:10:57 -04001660 files.update(OrderedDict((os.path.join(patches_dir, key), val) for
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001661 key, val in list(upd_p.items()) + list(new_p.items())))
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001662 if files or remove_files:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001663 removevalues = None
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001664 if remove_files:
1665 removedentries, remaining = _remove_file_entries(
1666 srcuri, remove_files)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001667 if removedentries or remaining:
1668 remaining = ['file://' + os.path.basename(item) for
1669 item in remaining]
1670 removevalues = {'SRC_URI': removedentries + remaining}
Brad Bishop316dfdd2018-06-25 12:45:53 -04001671 appendfile, destpath = oe.recipeutils.bbappend_recipe(
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001672 rd, appendlayerdir, files,
1673 wildcardver=wildcard_version,
Brad Bishop316dfdd2018-06-25 12:45:53 -04001674 removevalues=removevalues,
1675 redirect_output=dry_run_outdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001676 else:
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001677 logger.info('No patches or local source files needed updating')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001678 else:
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001679 # Update existing files
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001680 files_dir = _determine_files_dir(rd)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001681 for basepath, path in upd_f.items():
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001682 logger.info('Updating file %s' % basepath)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001683 if os.path.isabs(basepath):
1684 # Original file (probably with subdir pointing inside source tree)
1685 # so we do not want to move it, just copy
Brad Bishop316dfdd2018-06-25 12:45:53 -04001686 _copy_file(basepath, path,
1687 dry_run_outdir=dry_run_outdir, base_outdir=recipedir)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001688 else:
Brad Bishop316dfdd2018-06-25 12:45:53 -04001689 _move_file(os.path.join(local_files_dir, basepath), path,
1690 dry_run_outdir=dry_run_outdir, base_outdir=recipedir)
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001691 updatefiles = True
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001692 for basepath, path in upd_p.items():
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001693 patchfn = os.path.join(patches_dir, basepath)
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001694 if os.path.dirname(path) + '/' == dl_dir:
1695 # This is a a downloaded patch file - we now need to
1696 # replace the entry in SRC_URI with our local version
1697 logger.info('Replacing remote patch %s with updated local version' % basepath)
1698 path = os.path.join(files_dir, basepath)
1699 _replace_srcuri_entry(srcuri, basepath, 'file://%s' % basepath)
1700 updaterecipe = True
1701 else:
Brad Bishop316dfdd2018-06-25 12:45:53 -04001702 logger.info('Updating patch %s%s' % (basepath, dry_run_suffix))
1703 _move_file(patchfn, path,
1704 dry_run_outdir=dry_run_outdir, base_outdir=recipedir)
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001705 updatefiles = True
1706 # Add any new files
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001707 for basepath, path in new_f.items():
Brad Bishop316dfdd2018-06-25 12:45:53 -04001708 logger.info('Adding new file %s%s' % (basepath, dry_run_suffix))
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001709 _move_file(os.path.join(local_files_dir, basepath),
Brad Bishop316dfdd2018-06-25 12:45:53 -04001710 os.path.join(files_dir, basepath),
1711 dry_run_outdir=dry_run_outdir,
1712 base_outdir=recipedir)
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001713 srcuri.append('file://%s' % basepath)
1714 updaterecipe = True
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001715 for basepath, path in new_p.items():
Brad Bishop316dfdd2018-06-25 12:45:53 -04001716 logger.info('Adding new patch %s%s' % (basepath, dry_run_suffix))
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001717 _move_file(os.path.join(patches_dir, basepath),
Brad Bishop316dfdd2018-06-25 12:45:53 -04001718 os.path.join(files_dir, basepath),
1719 dry_run_outdir=dry_run_outdir,
1720 base_outdir=recipedir)
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001721 srcuri.append('file://%s' % basepath)
1722 updaterecipe = True
1723 # Update recipe, if needed
1724 if _remove_file_entries(srcuri, remove_files)[0]:
1725 updaterecipe = True
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001726 if updaterecipe:
Brad Bishop316dfdd2018-06-25 12:45:53 -04001727 if not dry_run_outdir:
1728 logger.info('Updating recipe %s' % os.path.basename(recipefile))
1729 ret = oe.recipeutils.patch_recipe(rd, recipefile,
1730 {'SRC_URI': ' '.join(srcuri)},
1731 redirect_output=dry_run_outdir)
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001732 elif not updatefiles:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001733 # Neither patches nor recipe were updated
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001734 logger.info('No patches or files need updating')
Brad Bishop316dfdd2018-06-25 12:45:53 -04001735 return False, None, []
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001736 finally:
1737 shutil.rmtree(tempdir)
1738
Brad Bishop316dfdd2018-06-25 12:45:53 -04001739 _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove, dry_run=dry_run_outdir)
1740 return True, appendfile, remove_files
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001741
1742def _guess_recipe_update_mode(srctree, rdata):
1743 """Guess the recipe update mode to use"""
Andrew Geisslerc9f78652020-09-18 14:11:35 -05001744 src_uri = (rdata.getVar('SRC_URI') or '').split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001745 git_uris = [uri for uri in src_uri if uri.startswith('git://')]
1746 if not git_uris:
1747 return 'patch'
1748 # Just use the first URI for now
1749 uri = git_uris[0]
1750 # Check remote branch
1751 params = bb.fetch.decodeurl(uri)[5]
1752 upstr_branch = params['branch'] if 'branch' in params else 'master'
1753 # Check if current branch HEAD is found in upstream branch
1754 stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree)
1755 head_rev = stdout.rstrip()
1756 stdout, _ = bb.process.run('git branch -r --contains %s' % head_rev,
1757 cwd=srctree)
1758 remote_brs = [branch.strip() for branch in stdout.splitlines()]
1759 if 'origin/' + upstr_branch in remote_brs:
1760 return 'srcrev'
1761
1762 return 'patch'
1763
Brad Bishop316dfdd2018-06-25 12:45:53 -04001764def _update_recipe(recipename, workspace, rd, mode, appendlayerdir, wildcard_version, no_remove, initial_rev, no_report_remove=False, dry_run_outdir=None, no_overrides=False, force_patch_refresh=False):
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001765 srctree = workspace[recipename]['srctree']
1766 if mode == 'auto':
1767 mode = _guess_recipe_update_mode(srctree, rd)
1768
Brad Bishop316dfdd2018-06-25 12:45:53 -04001769 override_branches = []
1770 mainbranch = None
1771 startbranch = None
1772 if not no_overrides:
1773 stdout, _ = bb.process.run('git branch', cwd=srctree)
1774 other_branches = []
1775 for line in stdout.splitlines():
1776 branchname = line[2:]
1777 if line.startswith('* '):
1778 startbranch = branchname
1779 if branchname.startswith(override_branch_prefix):
1780 override_branches.append(branchname)
1781 else:
1782 other_branches.append(branchname)
1783
1784 if override_branches:
1785 logger.debug('_update_recipe: override branches: %s' % override_branches)
1786 logger.debug('_update_recipe: other branches: %s' % other_branches)
1787 if startbranch.startswith(override_branch_prefix):
1788 if len(other_branches) == 1:
1789 mainbranch = other_branches[1]
1790 else:
1791 raise DevtoolError('Unable to determine main branch - please check out the main branch in source tree first')
1792 else:
1793 mainbranch = startbranch
1794
1795 checkedout = None
1796 anyupdated = False
1797 appendfile = None
1798 allremoved = []
1799 if override_branches:
1800 logger.info('Handling main branch (%s)...' % mainbranch)
1801 if startbranch != mainbranch:
1802 bb.process.run('git checkout %s' % mainbranch, cwd=srctree)
1803 checkedout = mainbranch
1804 try:
1805 branchlist = [mainbranch] + override_branches
1806 for branch in branchlist:
1807 crd = bb.data.createCopy(rd)
1808 if branch != mainbranch:
1809 logger.info('Handling branch %s...' % branch)
1810 override = branch[len(override_branch_prefix):]
1811 crd.appendVar('OVERRIDES', ':%s' % override)
1812 bb.process.run('git checkout %s' % branch, cwd=srctree)
1813 checkedout = branch
1814
1815 if mode == 'srcrev':
1816 updated, appendf, removed = _update_recipe_srcrev(recipename, workspace, srctree, crd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir)
1817 elif mode == 'patch':
1818 updated, appendf, removed = _update_recipe_patch(recipename, workspace, srctree, crd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev, dry_run_outdir, force_patch_refresh)
1819 else:
1820 raise DevtoolError('update_recipe: invalid mode %s' % mode)
1821 if updated:
1822 anyupdated = True
1823 if appendf:
1824 appendfile = appendf
1825 allremoved.extend(removed)
1826 finally:
1827 if startbranch and checkedout != startbranch:
1828 bb.process.run('git checkout %s' % startbranch, cwd=srctree)
1829
1830 return anyupdated, appendfile, allremoved
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001831
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001832def update_recipe(args, config, basepath, workspace):
1833 """Entry point for the devtool 'update-recipe' subcommand"""
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001834 check_workspace_recipe(workspace, args.recipename)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001835
1836 if args.append:
1837 if not os.path.exists(args.append):
1838 raise DevtoolError('bbappend destination layer directory "%s" '
1839 'does not exist' % args.append)
1840 if not os.path.exists(os.path.join(args.append, 'conf', 'layer.conf')):
1841 raise DevtoolError('conf/layer.conf not found in bbappend '
1842 'destination layer "%s"' % args.append)
1843
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001844 tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001845 try:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001846
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001847 rd = parse_recipe(config, tinfoil, args.recipename, True)
1848 if not rd:
1849 return 1
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001850
Brad Bishop316dfdd2018-06-25 12:45:53 -04001851 dry_run_output = None
1852 dry_run_outdir = None
1853 if args.dry_run:
1854 dry_run_output = tempfile.TemporaryDirectory(prefix='devtool')
1855 dry_run_outdir = dry_run_output.name
1856 updated, _, _ = _update_recipe(args.recipename, workspace, rd, args.mode, args.append, args.wildcard_version, args.no_remove, args.initial_rev, dry_run_outdir=dry_run_outdir, no_overrides=args.no_overrides, force_patch_refresh=args.force_patch_refresh)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001857
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001858 if updated:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001859 rf = rd.getVar('FILE')
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001860 if rf.startswith(config.workspace_path):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001861 logger.warning('Recipe file %s has been updated but is inside the workspace - you will need to move it (and any associated files next to it) out to the desired layer before using "devtool reset" in order to keep any changes' % rf)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001862 finally:
1863 tinfoil.shutdown()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001864
1865 return 0
1866
1867
1868def status(args, config, basepath, workspace):
1869 """Entry point for the devtool 'status' subcommand"""
1870 if workspace:
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001871 for recipe, value in sorted(workspace.items()):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001872 recipefile = value['recipefile']
1873 if recipefile:
1874 recipestr = ' (%s)' % recipefile
1875 else:
1876 recipestr = ''
1877 print("%s: %s%s" % (recipe, value['srctree'], recipestr))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001878 else:
1879 logger.info('No recipes currently in your workspace - you can use "devtool modify" to work on an existing recipe or "devtool add" to add a new one')
1880 return 0
1881
1882
Brad Bishop64c979e2019-11-04 13:55:29 -05001883def _reset(recipes, no_clean, remove_work, config, basepath, workspace):
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001884 """Reset one or more recipes"""
Brad Bishop316dfdd2018-06-25 12:45:53 -04001885 import oe.path
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001886
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001887 def clean_preferred_provider(pn, layerconf_path):
1888 """Remove PREFERRED_PROVIDER from layer.conf'"""
1889 import re
1890 layerconf_file = os.path.join(layerconf_path, 'conf', 'layer.conf')
1891 new_layerconf_file = os.path.join(layerconf_path, 'conf', '.layer.conf')
1892 pprovider_found = False
1893 with open(layerconf_file, 'r') as f:
1894 lines = f.readlines()
1895 with open(new_layerconf_file, 'a') as nf:
1896 for line in lines:
1897 pprovider_exp = r'^PREFERRED_PROVIDER_.*? = "' + pn + r'"$'
1898 if not re.match(pprovider_exp, line):
1899 nf.write(line)
1900 else:
1901 pprovider_found = True
1902 if pprovider_found:
1903 shutil.move(new_layerconf_file, layerconf_file)
1904 else:
1905 os.remove(new_layerconf_file)
1906
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001907 if recipes and not no_clean:
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001908 if len(recipes) == 1:
1909 logger.info('Cleaning sysroot for recipe %s...' % recipes[0])
1910 else:
1911 logger.info('Cleaning sysroot for recipes %s...' % ', '.join(recipes))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001912 # If the recipe file itself was created in the workspace, and
1913 # it uses BBCLASSEXTEND, then we need to also clean the other
1914 # variants
1915 targets = []
1916 for recipe in recipes:
1917 targets.append(recipe)
1918 recipefile = workspace[recipe]['recipefile']
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001919 if recipefile and os.path.exists(recipefile):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001920 targets.extend(get_bbclassextend_targets(recipefile, recipe))
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001921 try:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001922 exec_build_env_command(config.init_path, basepath, 'bitbake -c clean %s' % ' '.join(targets))
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001923 except bb.process.ExecutionError as e:
1924 raise DevtoolError('Command \'%s\' failed, output:\n%s\nIf you '
1925 'wish, you may specify -n/--no-clean to '
1926 'skip running this command when resetting' %
1927 (e.command, e.stdout))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001928
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001929 for pn in recipes:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001930 _check_preserve(config, pn)
1931
Brad Bishop316dfdd2018-06-25 12:45:53 -04001932 appendfile = workspace[pn]['bbappend']
1933 if os.path.exists(appendfile):
1934 # This shouldn't happen, but is possible if devtool errored out prior to
1935 # writing the md5 file. We need to delete this here or the recipe won't
1936 # actually be reset
1937 os.remove(appendfile)
1938
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001939 preservepath = os.path.join(config.workspace_path, 'attic', pn, pn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001940 def preservedir(origdir):
1941 if os.path.exists(origdir):
1942 for root, dirs, files in os.walk(origdir):
1943 for fn in files:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001944 logger.warning('Preserving %s in %s' % (fn, preservepath))
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001945 _move_file(os.path.join(origdir, fn),
1946 os.path.join(preservepath, fn))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001947 for dn in dirs:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001948 preservedir(os.path.join(root, dn))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001949 os.rmdir(origdir)
1950
Brad Bishop316dfdd2018-06-25 12:45:53 -04001951 recipefile = workspace[pn]['recipefile']
1952 if recipefile and oe.path.is_path_parent(config.workspace_path, recipefile):
1953 # This should always be true if recipefile is set, but just in case
1954 preservedir(os.path.dirname(recipefile))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001955 # We don't automatically create this dir next to appends, but the user can
1956 preservedir(os.path.join(config.workspace_path, 'appends', pn))
1957
Brad Bishop316dfdd2018-06-25 12:45:53 -04001958 srctreebase = workspace[pn]['srctreebase']
1959 if os.path.isdir(srctreebase):
1960 if os.listdir(srctreebase):
Brad Bishop64c979e2019-11-04 13:55:29 -05001961 if remove_work:
1962 logger.info('-r argument used on %s, removing source tree.'
1963 ' You will lose any unsaved work' %pn)
1964 shutil.rmtree(srctreebase)
1965 else:
1966 # We don't want to risk wiping out any work in progress
1967 logger.info('Leaving source tree %s as-is; if you no '
1968 'longer need it then please delete it manually'
1969 % srctreebase)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001970 else:
1971 # This is unlikely, but if it's empty we can just remove it
Brad Bishop316dfdd2018-06-25 12:45:53 -04001972 os.rmdir(srctreebase)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001973
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001974 clean_preferred_provider(pn, config.workspace_path)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001975
1976def reset(args, config, basepath, workspace):
1977 """Entry point for the devtool 'reset' subcommand"""
1978 import bb
Brad Bishop64c979e2019-11-04 13:55:29 -05001979 import shutil
1980
1981 recipes = ""
1982
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001983 if args.recipename:
1984 if args.all:
1985 raise DevtoolError("Recipe cannot be specified if -a/--all is used")
1986 else:
1987 for recipe in args.recipename:
1988 check_workspace_recipe(workspace, recipe, checksrc=False)
1989 elif not args.all:
1990 raise DevtoolError("Recipe must be specified, or specify -a/--all to "
1991 "reset all recipes")
1992 if args.all:
1993 recipes = list(workspace.keys())
1994 else:
1995 recipes = args.recipename
1996
Brad Bishop64c979e2019-11-04 13:55:29 -05001997 _reset(recipes, args.no_clean, args.remove_work, config, basepath, workspace)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001998
1999 return 0
2000
2001
2002def _get_layer(layername, d):
2003 """Determine the base layer path for the specified layer name/path"""
Brad Bishop6e60e8b2018-02-01 10:27:11 -05002004 layerdirs = d.getVar('BBLAYERS').split()
Brad Bishop96ff1982019-08-19 13:50:42 -04002005 layers = {} # {basename: layer_paths}
2006 for p in layerdirs:
2007 bn = os.path.basename(p)
2008 if bn not in layers:
2009 layers[bn] = [p]
2010 else:
2011 layers[bn].append(p)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06002012 # Provide some shortcuts
2013 if layername.lower() in ['oe-core', 'openembedded-core']:
Brad Bishop96ff1982019-08-19 13:50:42 -04002014 layername = 'meta'
2015 layer_paths = layers.get(layername, None)
2016 if not layer_paths:
2017 return os.path.abspath(layername)
2018 elif len(layer_paths) == 1:
2019 return os.path.abspath(layer_paths[0])
Patrick Williamsc0f7c042017-02-23 20:41:17 -06002020 else:
Brad Bishop96ff1982019-08-19 13:50:42 -04002021 # multiple layers having the same base name
2022 logger.warning("Multiple layers have the same base name '%s', use the first one '%s'." % (layername, layer_paths[0]))
2023 logger.warning("Consider using path instead of base name to specify layer:\n\t\t%s" % '\n\t\t'.join(layer_paths))
2024 return os.path.abspath(layer_paths[0])
2025
Patrick Williamsc0f7c042017-02-23 20:41:17 -06002026
2027def finish(args, config, basepath, workspace):
2028 """Entry point for the devtool 'finish' subcommand"""
2029 import bb
2030 import oe.recipeutils
2031
2032 check_workspace_recipe(workspace, args.recipename)
2033
Brad Bishop316dfdd2018-06-25 12:45:53 -04002034 dry_run_suffix = ' (dry-run)' if args.dry_run else ''
2035
2036 # Grab the equivalent of COREBASE without having to initialise tinfoil
2037 corebasedir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..'))
2038
2039 srctree = workspace[args.recipename]['srctree']
2040 check_git_repo_op(srctree, [corebasedir])
2041 dirty = check_git_repo_dirty(srctree)
2042 if dirty:
2043 if args.force:
2044 logger.warning('Source tree is not clean, continuing as requested by -f/--force')
2045 else:
2046 raise DevtoolError('Source tree is not clean:\n\n%s\nEnsure you have committed your changes or use -f/--force if you are sure there\'s nothing that needs to be committed' % dirty)
2047
Brad Bishop00e122a2019-10-05 11:10:57 -04002048 no_clean = args.no_clean
Brad Bishop64c979e2019-11-04 13:55:29 -05002049 remove_work=args.remove_work
Patrick Williamsc0f7c042017-02-23 20:41:17 -06002050 tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
2051 try:
Brad Bishop6dbb3162019-11-25 09:41:34 -05002052 rd = parse_recipe(config, tinfoil, args.recipename, True)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06002053 if not rd:
2054 return 1
2055
2056 destlayerdir = _get_layer(args.destination, tinfoil.config_data)
Brad Bishop316dfdd2018-06-25 12:45:53 -04002057 recipefile = rd.getVar('FILE')
2058 recipedir = os.path.dirname(recipefile)
2059 origlayerdir = oe.recipeutils.find_layerdir(recipefile)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06002060
2061 if not os.path.isdir(destlayerdir):
2062 raise DevtoolError('Unable to find layer or directory matching "%s"' % args.destination)
2063
2064 if os.path.abspath(destlayerdir) == config.workspace_path:
2065 raise DevtoolError('"%s" specifies the workspace layer - that is not a valid destination' % args.destination)
2066
2067 # If it's an upgrade, grab the original path
2068 origpath = None
2069 origfilelist = None
2070 append = workspace[args.recipename]['bbappend']
2071 with open(append, 'r') as f:
2072 for line in f:
2073 if line.startswith('# original_path:'):
2074 origpath = line.split(':')[1].strip()
2075 elif line.startswith('# original_files:'):
2076 origfilelist = line.split(':')[1].split()
2077
Brad Bishop316dfdd2018-06-25 12:45:53 -04002078 destlayerbasedir = oe.recipeutils.find_layerdir(destlayerdir)
2079
Patrick Williamsc0f7c042017-02-23 20:41:17 -06002080 if origlayerdir == config.workspace_path:
2081 # Recipe file itself is in workspace, update it there first
2082 appendlayerdir = None
2083 origrelpath = None
2084 if origpath:
2085 origlayerpath = oe.recipeutils.find_layerdir(origpath)
2086 if origlayerpath:
2087 origrelpath = os.path.relpath(origpath, origlayerpath)
2088 destpath = oe.recipeutils.get_bbfile_path(rd, destlayerdir, origrelpath)
2089 if not destpath:
2090 raise DevtoolError("Unable to determine destination layer path - check that %s specifies an actual layer and %s/conf/layer.conf specifies BBFILES. You may also need to specify a more complete path." % (args.destination, destlayerdir))
Brad Bishop6e60e8b2018-02-01 10:27:11 -05002091 # Warn if the layer isn't in bblayers.conf (the code to create a bbappend will do this in other cases)
2092 layerdirs = [os.path.abspath(layerdir) for layerdir in rd.getVar('BBLAYERS').split()]
Brad Bishop316dfdd2018-06-25 12:45:53 -04002093 if not os.path.abspath(destlayerbasedir) in layerdirs:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05002094 bb.warn('Specified destination layer is not currently enabled in bblayers.conf, so the %s recipe will now be unavailable in your current configuration until you add the layer there' % args.recipename)
2095
Patrick Williamsc0f7c042017-02-23 20:41:17 -06002096 elif destlayerdir == origlayerdir:
2097 # Same layer, update the original recipe
2098 appendlayerdir = None
2099 destpath = None
2100 else:
2101 # Create/update a bbappend in the specified layer
2102 appendlayerdir = destlayerdir
2103 destpath = None
2104
Brad Bishop316dfdd2018-06-25 12:45:53 -04002105 # Actually update the recipe / bbappend
2106 removing_original = (origpath and origfilelist and oe.recipeutils.find_layerdir(origpath) == destlayerbasedir)
2107 dry_run_output = None
2108 dry_run_outdir = None
2109 if args.dry_run:
2110 dry_run_output = tempfile.TemporaryDirectory(prefix='devtool')
2111 dry_run_outdir = dry_run_output.name
2112 updated, appendfile, removed = _update_recipe(args.recipename, workspace, rd, args.mode, appendlayerdir, wildcard_version=True, no_remove=False, no_report_remove=removing_original, initial_rev=args.initial_rev, dry_run_outdir=dry_run_outdir, no_overrides=args.no_overrides, force_patch_refresh=args.force_patch_refresh)
2113 removed = [os.path.relpath(pth, recipedir) for pth in removed]
2114
Patrick Williamsc0f7c042017-02-23 20:41:17 -06002115 # Remove any old files in the case of an upgrade
Brad Bishop316dfdd2018-06-25 12:45:53 -04002116 if removing_original:
Patrick Williamsc0f7c042017-02-23 20:41:17 -06002117 for fn in origfilelist:
2118 fnp = os.path.join(origpath, fn)
Brad Bishop316dfdd2018-06-25 12:45:53 -04002119 if fn in removed or not os.path.exists(os.path.join(recipedir, fn)):
2120 logger.info('Removing file %s%s' % (fnp, dry_run_suffix))
2121 if not args.dry_run:
2122 try:
2123 os.remove(fnp)
2124 except FileNotFoundError:
2125 pass
Patrick Williamsc0f7c042017-02-23 20:41:17 -06002126
2127 if origlayerdir == config.workspace_path and destpath:
2128 # Recipe file itself is in the workspace - need to move it and any
2129 # associated files to the specified layer
Brad Bishop6e60e8b2018-02-01 10:27:11 -05002130 no_clean = True
Brad Bishop316dfdd2018-06-25 12:45:53 -04002131 logger.info('Moving recipe file to %s%s' % (destpath, dry_run_suffix))
Patrick Williamsc0f7c042017-02-23 20:41:17 -06002132 for root, _, files in os.walk(recipedir):
2133 for fn in files:
2134 srcpath = os.path.join(root, fn)
2135 relpth = os.path.relpath(os.path.dirname(srcpath), recipedir)
2136 destdir = os.path.abspath(os.path.join(destpath, relpth))
Brad Bishop316dfdd2018-06-25 12:45:53 -04002137 destfp = os.path.join(destdir, fn)
2138 _move_file(srcpath, destfp, dry_run_outdir=dry_run_outdir, base_outdir=destpath)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06002139
Brad Bishop316dfdd2018-06-25 12:45:53 -04002140 if dry_run_outdir:
2141 import difflib
2142 comparelist = []
2143 for root, _, files in os.walk(dry_run_outdir):
2144 for fn in files:
2145 outf = os.path.join(root, fn)
2146 relf = os.path.relpath(outf, dry_run_outdir)
2147 logger.debug('dry-run: output file %s' % relf)
2148 if fn.endswith('.bb'):
2149 if origfilelist and origpath and destpath:
2150 # Need to match this up with the pre-upgrade recipe file
2151 for origf in origfilelist:
2152 if origf.endswith('.bb'):
2153 comparelist.append((os.path.abspath(os.path.join(origpath, origf)),
2154 outf,
2155 os.path.abspath(os.path.join(destpath, relf))))
2156 break
2157 else:
2158 # Compare to the existing recipe
2159 comparelist.append((recipefile, outf, recipefile))
2160 elif fn.endswith('.bbappend'):
2161 if appendfile:
2162 if os.path.exists(appendfile):
2163 comparelist.append((appendfile, outf, appendfile))
2164 else:
2165 comparelist.append((None, outf, appendfile))
2166 else:
2167 if destpath:
2168 recipedest = destpath
2169 elif appendfile:
2170 recipedest = os.path.dirname(appendfile)
2171 else:
2172 recipedest = os.path.dirname(recipefile)
2173 destfp = os.path.join(recipedest, relf)
2174 if os.path.exists(destfp):
2175 comparelist.append((destfp, outf, destfp))
2176 output = ''
2177 for oldfile, newfile, newfileshow in comparelist:
2178 if oldfile:
2179 with open(oldfile, 'r') as f:
2180 oldlines = f.readlines()
2181 else:
2182 oldfile = '/dev/null'
2183 oldlines = []
2184 with open(newfile, 'r') as f:
2185 newlines = f.readlines()
2186 if not newfileshow:
2187 newfileshow = newfile
2188 diff = difflib.unified_diff(oldlines, newlines, oldfile, newfileshow)
2189 difflines = list(diff)
2190 if difflines:
2191 output += ''.join(difflines)
2192 if output:
2193 logger.info('Diff of changed files:\n%s' % output)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06002194 finally:
2195 tinfoil.shutdown()
2196
2197 # Everything else has succeeded, we can now reset
Brad Bishop316dfdd2018-06-25 12:45:53 -04002198 if args.dry_run:
2199 logger.info('Resetting recipe (dry-run)')
2200 else:
Brad Bishop64c979e2019-11-04 13:55:29 -05002201 _reset([args.recipename], no_clean=no_clean, remove_work=remove_work, config=config, basepath=basepath, workspace=workspace)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06002202
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002203 return 0
2204
2205
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05002206def get_default_srctree(config, recipename=''):
2207 """Get the default srctree path"""
2208 srctreeparent = config.get('General', 'default_source_parent_dir', config.workspace_path)
2209 if recipename:
2210 return os.path.join(srctreeparent, 'sources', recipename)
2211 else:
2212 return os.path.join(srctreeparent, 'sources')
2213
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002214def register_commands(subparsers, context):
2215 """Register devtool subcommands from this plugin"""
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05002216
2217 defsrctree = get_default_srctree(context.config)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002218 parser_add = subparsers.add_parser('add', help='Add a new recipe',
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05002219 description='Adds a new recipe to the workspace to build a specified source tree. Can optionally fetch a remote URI and unpack it to create the source tree.',
2220 group='starting', order=100)
2221 parser_add.add_argument('recipename', nargs='?', help='Name for new recipe to add (just name - no version, path or extension). If not specified, will attempt to auto-detect it.')
2222 parser_add.add_argument('srctree', nargs='?', help='Path to external source tree. If not specified, a subdirectory of %s will be used.' % defsrctree)
2223 parser_add.add_argument('fetchuri', nargs='?', help='Fetch the specified URI and extract it to create the source tree')
Patrick Williamsf1e5d692016-03-30 15:21:19 -05002224 group = parser_add.add_mutually_exclusive_group()
2225 group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true")
2226 group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05002227 parser_add.add_argument('--fetch', '-f', help='Fetch the specified URI and extract it to create the source tree (deprecated - pass as positional argument instead)', metavar='URI')
Andrew Geissler82c905d2020-04-13 13:39:40 -05002228 parser_add.add_argument('--npm-dev', help='For npm, also fetch devDependencies', action="store_true")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002229 parser_add.add_argument('--version', '-V', help='Version to use within recipe (PV)')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05002230 parser_add.add_argument('--no-git', '-g', help='If fetching source, do not set up source tree as a git repository', action="store_true")
Brad Bishopd7bf8c12018-02-25 22:55:05 -05002231 group = parser_add.add_mutually_exclusive_group()
2232 group.add_argument('--srcrev', '-S', help='Source revision to fetch if fetching from an SCM such as git (default latest)')
2233 group.add_argument('--autorev', '-a', help='When fetching from a git repository, set SRCREV in the recipe to a floating revision instead of fixed', action="store_true")
2234 parser_add.add_argument('--srcbranch', '-B', help='Branch in source repository if fetching from an SCM such as git (default master)')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05002235 parser_add.add_argument('--binary', '-b', help='Treat the source tree as something that should be installed verbatim (no compilation, same directory structure). Useful with binary packages e.g. RPMs.', action='store_true')
2236 parser_add.add_argument('--also-native', help='Also add native variant (i.e. support building recipe for the build host as well as the target machine)', action='store_true')
2237 parser_add.add_argument('--src-subdir', help='Specify subdirectory within source tree to use', metavar='SUBDIR')
Brad Bishopd7bf8c12018-02-25 22:55:05 -05002238 parser_add.add_argument('--mirrors', help='Enable PREMIRRORS and MIRRORS for source tree fetching (disable by default).', action="store_true")
2239 parser_add.add_argument('--provides', '-p', help='Specify an alias for the item provided by the recipe. E.g. virtual/libgl')
Patrick Williamsc0f7c042017-02-23 20:41:17 -06002240 parser_add.set_defaults(func=add, fixed_setup=context.fixed_setup)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002241
2242 parser_modify = subparsers.add_parser('modify', help='Modify the source for an existing recipe',
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05002243 description='Sets up the build environment to modify the source for an existing recipe. The default behaviour is to extract the source being fetched by the recipe into a git tree so you can work on it; alternatively if you already have your own pre-prepared source tree you can specify -n/--no-extract.',
2244 group='starting', order=90)
2245 parser_modify.add_argument('recipename', help='Name of existing recipe to edit (just name - no version, path or extension)')
2246 parser_modify.add_argument('srctree', nargs='?', help='Path to external source tree. If not specified, a subdirectory of %s will be used.' % defsrctree)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002247 parser_modify.add_argument('--wildcard', '-w', action="store_true", help='Use wildcard for unversioned bbappend')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05002248 group = parser_modify.add_mutually_exclusive_group()
2249 group.add_argument('--extract', '-x', action="store_true", help='Extract source for recipe (default)')
2250 group.add_argument('--no-extract', '-n', action="store_true", help='Do not extract source, expect it to exist')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002251 group = parser_modify.add_mutually_exclusive_group()
2252 group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true")
2253 group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05002254 parser_modify.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (when not using -n/--no-extract) (default "%(default)s")')
Brad Bishop316dfdd2018-06-25 12:45:53 -04002255 parser_modify.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations')
Brad Bishop6e60e8b2018-02-01 10:27:11 -05002256 parser_modify.add_argument('--keep-temp', help='Keep temporary directory (for debugging)', action="store_true")
Brad Bishopd7bf8c12018-02-25 22:55:05 -05002257 parser_modify.set_defaults(func=modify, fixed_setup=context.fixed_setup)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002258
2259 parser_extract = subparsers.add_parser('extract', help='Extract the source for an existing recipe',
2260 description='Extracts the source for an existing recipe',
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05002261 group='advanced')
2262 parser_extract.add_argument('recipename', help='Name of recipe to extract the source for')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002263 parser_extract.add_argument('srctree', help='Path to where to extract the source tree')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05002264 parser_extract.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (default "%(default)s")')
Brad Bishop316dfdd2018-06-25 12:45:53 -04002265 parser_extract.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002266 parser_extract.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
Brad Bishopd7bf8c12018-02-25 22:55:05 -05002267 parser_extract.set_defaults(func=extract, fixed_setup=context.fixed_setup)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05002268
2269 parser_sync = subparsers.add_parser('sync', help='Synchronize the source tree for an existing recipe',
2270 description='Synchronize the previously extracted source tree for an existing recipe',
2271 formatter_class=argparse.ArgumentDefaultsHelpFormatter,
2272 group='advanced')
2273 parser_sync.add_argument('recipename', help='Name of recipe to sync the source for')
2274 parser_sync.add_argument('srctree', help='Path to the source tree')
2275 parser_sync.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout')
2276 parser_sync.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
Brad Bishopd7bf8c12018-02-25 22:55:05 -05002277 parser_sync.set_defaults(func=sync, fixed_setup=context.fixed_setup)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002278
Brad Bishop6e60e8b2018-02-01 10:27:11 -05002279 parser_rename = subparsers.add_parser('rename', help='Rename a recipe file in the workspace',
2280 description='Renames the recipe file for a recipe in the workspace, changing the name or version part or both, ensuring that all references within the workspace are updated at the same time. Only works when the recipe file itself is in the workspace, e.g. after devtool add. Particularly useful when devtool add did not automatically determine the correct name.',
2281 group='working', order=10)
2282 parser_rename.add_argument('recipename', help='Current name of recipe to rename')
2283 parser_rename.add_argument('newname', nargs='?', help='New name for recipe (optional, not needed if you only want to change the version)')
2284 parser_rename.add_argument('--version', '-V', help='Change the version (NOTE: this does not change the version fetched by the recipe, just the version in the recipe file name)')
2285 parser_rename.add_argument('--no-srctree', '-s', action='store_true', help='Do not rename the source tree directory (if the default source tree path has been used) - keeping the old name may be desirable if there are internal/other external references to this path')
2286 parser_rename.set_defaults(func=rename)
2287
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002288 parser_update_recipe = subparsers.add_parser('update-recipe', help='Apply changes from external source tree to recipe',
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05002289 description='Applies changes from external source tree to a recipe (updating/adding/removing patches as necessary, or by updating SRCREV). Note that these changes need to have been committed to the git repository in order to be recognised.',
2290 group='working', order=-90)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002291 parser_update_recipe.add_argument('recipename', help='Name of recipe to update')
2292 parser_update_recipe.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05002293 parser_update_recipe.add_argument('--initial-rev', help='Override starting revision for patches')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002294 parser_update_recipe.add_argument('--append', '-a', help='Write changes to a bbappend in the specified layer instead of the recipe', metavar='LAYERDIR')
2295 parser_update_recipe.add_argument('--wildcard-version', '-w', help='In conjunction with -a/--append, use a wildcard to make the bbappend apply to any recipe version', action='store_true')
2296 parser_update_recipe.add_argument('--no-remove', '-n', action="store_true", help='Don\'t remove patches, only add or update')
Brad Bishop316dfdd2018-06-25 12:45:53 -04002297 parser_update_recipe.add_argument('--no-overrides', '-O', action="store_true", help='Do not handle other override branches (if they exist)')
2298 parser_update_recipe.add_argument('--dry-run', '-N', action="store_true", help='Dry-run (just report changes instead of writing them)')
2299 parser_update_recipe.add_argument('--force-patch-refresh', action="store_true", help='Update patches in the layer even if they have not been modified (useful for refreshing patch context)')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002300 parser_update_recipe.set_defaults(func=update_recipe)
2301
2302 parser_status = subparsers.add_parser('status', help='Show workspace status',
2303 description='Lists recipes currently in your workspace and the paths to their respective external source trees',
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05002304 group='info', order=100)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002305 parser_status.set_defaults(func=status)
2306
2307 parser_reset = subparsers.add_parser('reset', help='Remove a recipe from your workspace',
Patrick Williamsc0f7c042017-02-23 20:41:17 -06002308 description='Removes the specified recipe(s) from your workspace (resetting its state back to that defined by the metadata).',
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05002309 group='working', order=-100)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06002310 parser_reset.add_argument('recipename', nargs='*', help='Recipe to reset')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002311 parser_reset.add_argument('--all', '-a', action="store_true", help='Reset all recipes (clear workspace)')
2312 parser_reset.add_argument('--no-clean', '-n', action="store_true", help='Don\'t clean the sysroot to remove recipe output')
Brad Bishop64c979e2019-11-04 13:55:29 -05002313 parser_reset.add_argument('--remove-work', '-r', action="store_true", help='Clean the sources directory along with append')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002314 parser_reset.set_defaults(func=reset)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06002315
2316 parser_finish = subparsers.add_parser('finish', help='Finish working on a recipe in your workspace',
Brad Bishop316dfdd2018-06-25 12:45:53 -04002317 description='Pushes any committed changes to the specified recipe to the specified layer and removes it from your workspace. Roughly equivalent to an update-recipe followed by reset, except the update-recipe step will do the "right thing" depending on the recipe and the destination layer specified. Note that your changes must have been committed to the git repository in order to be recognised.',
Patrick Williamsc0f7c042017-02-23 20:41:17 -06002318 group='working', order=-100)
2319 parser_finish.add_argument('recipename', help='Recipe to finish')
2320 parser_finish.add_argument('destination', help='Layer/path to put recipe into. Can be the name of a layer configured in your bblayers.conf, the path to the base of a layer, or a partial path inside a layer. %(prog)s will attempt to complete the path based on the layer\'s structure.')
2321 parser_finish.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE')
2322 parser_finish.add_argument('--initial-rev', help='Override starting revision for patches')
Brad Bishop316dfdd2018-06-25 12:45:53 -04002323 parser_finish.add_argument('--force', '-f', action="store_true", help='Force continuing even if there are uncommitted changes in the source tree repository')
Brad Bishop64c979e2019-11-04 13:55:29 -05002324 parser_finish.add_argument('--remove-work', '-r', action="store_true", help='Clean the sources directory under workspace')
Brad Bishop00e122a2019-10-05 11:10:57 -04002325 parser_finish.add_argument('--no-clean', '-n', action="store_true", help='Don\'t clean the sysroot to remove recipe output')
Brad Bishop316dfdd2018-06-25 12:45:53 -04002326 parser_finish.add_argument('--no-overrides', '-O', action="store_true", help='Do not handle other override branches (if they exist)')
2327 parser_finish.add_argument('--dry-run', '-N', action="store_true", help='Dry-run (just report changes instead of writing them)')
2328 parser_finish.add_argument('--force-patch-refresh', action="store_true", help='Update patches in the layer even if they have not been modified (useful for refreshing patch context)')
Patrick Williamsc0f7c042017-02-23 20:41:17 -06002329 parser_finish.set_defaults(func=finish)