blob: 18c5b66a295f64bddb73bde6903907332c14ab17 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# Development tool - upgrade command plugin
2#
Brad Bishopd7bf8c12018-02-25 22:55:05 -05003# Copyright (C) 2014-2017 Intel Corporation
Patrick Williamsc124f4f2015-09-15 14:41:29 -05004#
Brad Bishopc342db32019-05-15 21:57:59 -04005# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -05006#
7"""Devtool upgrade plugin"""
8
9import os
10import sys
11import re
12import shutil
13import tempfile
14import logging
15import argparse
16import scriptutils
17import errno
18import bb
Brad Bishop6e60e8b2018-02-01 10:27:11 -050019
20devtool_path = os.path.dirname(os.path.realpath(__file__)) + '/../../../meta/lib'
21sys.path = sys.path + [devtool_path]
22
Patrick Williamsc124f4f2015-09-15 14:41:29 -050023import oe.recipeutils
24from devtool import standard
Brad Bishop316dfdd2018-06-25 12:45:53 -040025from devtool import exec_build_env_command, setup_tinfoil, DevtoolError, parse_recipe, use_external_build, update_unlockedsigs, check_prerelease_version
Patrick Williamsc124f4f2015-09-15 14:41:29 -050026
27logger = logging.getLogger('devtool')
28
Patrick Williamsc124f4f2015-09-15 14:41:29 -050029def _run(cmd, cwd=''):
30 logger.debug("Running command %s> %s" % (cwd,cmd))
31 return bb.process.run('%s' % cmd, cwd=cwd)
32
33def _get_srctree(tmpdir):
34 srctree = tmpdir
35 dirs = os.listdir(tmpdir)
36 if len(dirs) == 1:
37 srctree = os.path.join(tmpdir, dirs[0])
38 return srctree
39
40def _copy_source_code(orig, dest):
41 for path in standard._ls_tree(orig):
42 dest_dir = os.path.join(dest, os.path.dirname(path))
43 bb.utils.mkdirhier(dest_dir)
44 dest_path = os.path.join(dest, path)
Patrick Williamsf1e5d692016-03-30 15:21:19 -050045 shutil.move(os.path.join(orig, path), dest_path)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050046
Patrick Williamsc124f4f2015-09-15 14:41:29 -050047def _remove_patch_dirs(recipefolder):
48 for root, dirs, files in os.walk(recipefolder):
49 for d in dirs:
50 shutil.rmtree(os.path.join(root,d))
51
Patrick Williamsf1e5d692016-03-30 15:21:19 -050052def _recipe_contains(rd, var):
Brad Bishop6e60e8b2018-02-01 10:27:11 -050053 rf = rd.getVar('FILE')
Patrick Williamsf1e5d692016-03-30 15:21:19 -050054 varfiles = oe.recipeutils.get_var_files(rf, [var], rd)
Patrick Williamsc0f7c042017-02-23 20:41:17 -060055 for var, fn in varfiles.items():
Patrick Williamsf1e5d692016-03-30 15:21:19 -050056 if fn and fn.startswith(os.path.dirname(rf) + os.sep):
57 return True
58 return False
Patrick Williamsc124f4f2015-09-15 14:41:29 -050059
60def _rename_recipe_dirs(oldpv, newpv, path):
61 for root, dirs, files in os.walk(path):
Patrick Williamsc0f7c042017-02-23 20:41:17 -060062 # Rename directories with the version in their name
Patrick Williamsc124f4f2015-09-15 14:41:29 -050063 for olddir in dirs:
64 if olddir.find(oldpv) != -1:
65 newdir = olddir.replace(oldpv, newpv)
66 if olddir != newdir:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050067 shutil.move(os.path.join(path, olddir), os.path.join(path, newdir))
Patrick Williamsc0f7c042017-02-23 20:41:17 -060068 # Rename any inc files with the version in their name (unusual, but possible)
69 for oldfile in files:
70 if oldfile.endswith('.inc'):
71 if oldfile.find(oldpv) != -1:
72 newfile = oldfile.replace(oldpv, newpv)
73 if oldfile != newfile:
74 os.rename(os.path.join(path, oldfile), os.path.join(path, newfile))
Patrick Williamsc124f4f2015-09-15 14:41:29 -050075
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050076def _rename_recipe_file(oldrecipe, bpn, oldpv, newpv, path):
77 oldrecipe = os.path.basename(oldrecipe)
78 if oldrecipe.endswith('_%s.bb' % oldpv):
79 newrecipe = '%s_%s.bb' % (bpn, newpv)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050080 if oldrecipe != newrecipe:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050081 shutil.move(os.path.join(path, oldrecipe), os.path.join(path, newrecipe))
Patrick Williamsc124f4f2015-09-15 14:41:29 -050082 else:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050083 newrecipe = oldrecipe
Patrick Williamsc124f4f2015-09-15 14:41:29 -050084 return os.path.join(path, newrecipe)
85
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050086def _rename_recipe_files(oldrecipe, bpn, oldpv, newpv, path):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050087 _rename_recipe_dirs(oldpv, newpv, path)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050088 return _rename_recipe_file(oldrecipe, bpn, oldpv, newpv, path)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050089
Patrick Williamsc0f7c042017-02-23 20:41:17 -060090def _write_append(rc, srctree, same_dir, no_same_dir, rev, copied, workspace, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050091 """Writes an append file"""
92 if not os.path.exists(rc):
93 raise DevtoolError("bbappend not created because %s does not exist" % rc)
94
95 appendpath = os.path.join(workspace, 'appends')
96 if not os.path.exists(appendpath):
97 bb.utils.mkdirhier(appendpath)
98
99 brf = os.path.basename(os.path.splitext(rc)[0]) # rc basename
100
101 srctree = os.path.abspath(srctree)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500102 pn = d.getVar('PN')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500103 af = os.path.join(appendpath, '%s.bbappend' % brf)
104 with open(af, 'w') as f:
105 f.write('FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n\n')
106 f.write('inherit externalsrc\n')
107 f.write(('# NOTE: We use pn- overrides here to avoid affecting'
108 'multiple variants in the case where the recipe uses BBCLASSEXTEND\n'))
109 f.write('EXTERNALSRC_pn-%s = "%s"\n' % (pn, srctree))
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500110 b_is_s = use_external_build(same_dir, no_same_dir, d)
111 if b_is_s:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500112 f.write('EXTERNALSRC_BUILD_pn-%s = "%s"\n' % (pn, srctree))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600113 f.write('\n')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500114 if rev:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600115 f.write('# initial_rev: %s\n' % rev)
116 if copied:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500117 f.write('# original_path: %s\n' % os.path.dirname(d.getVar('FILE')))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600118 f.write('# original_files: %s\n' % ' '.join(copied))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500119 return af
120
121def _cleanup_on_error(rf, srctree):
122 rfp = os.path.split(rf)[0] # recipe folder
123 rfpp = os.path.split(rfp)[0] # recipes folder
124 if os.path.exists(rfp):
Brad Bishop96ff1982019-08-19 13:50:42 -0400125 shutil.rmtree(rfp)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500126 if not len(os.listdir(rfpp)):
127 os.rmdir(rfpp)
128 srctree = os.path.abspath(srctree)
129 if os.path.exists(srctree):
130 shutil.rmtree(srctree)
131
Brad Bishop96ff1982019-08-19 13:50:42 -0400132def _upgrade_error(e, rf, srctree, keep_failure=False, extramsg=None):
133 if rf and not keep_failure:
134 _cleanup_on_error(rf, srctree)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500135 logger.error(e)
Brad Bishop96ff1982019-08-19 13:50:42 -0400136 if extramsg:
137 logger.error(extramsg)
138 if keep_failure:
139 logger.info('Preserving failed upgrade files (--keep-failure)')
140 sys.exit(1)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500141
142def _get_uri(rd):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500143 srcuris = rd.getVar('SRC_URI').split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500144 if not len(srcuris):
145 raise DevtoolError('SRC_URI not found on recipe')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500146 # Get first non-local entry in SRC_URI - usually by convention it's
147 # the first entry, but not always!
148 srcuri = None
149 for entry in srcuris:
150 if not entry.startswith('file://'):
151 srcuri = entry
152 break
153 if not srcuri:
154 raise DevtoolError('Unable to find non-local entry in SRC_URI')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500155 srcrev = '${AUTOREV}'
156 if '://' in srcuri:
157 # Fetch a URL
158 rev_re = re.compile(';rev=([^;]+)')
159 res = rev_re.search(srcuri)
160 if res:
161 srcrev = res.group(1)
162 srcuri = rev_re.sub('', srcuri)
163 return srcuri, srcrev
164
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500165def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, keep_temp, tinfoil, rd):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500166 """Extract sources of a recipe with a new version"""
167
168 def __run(cmd):
169 """Simple wrapper which calls _run with srctree as cwd"""
170 return _run(cmd, srctree)
171
172 crd = rd.createCopy()
173
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500174 pv = crd.getVar('PV')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500175 crd.setVar('PV', newpv)
176
177 tmpsrctree = None
178 uri, rev = _get_uri(crd)
179 if srcrev:
180 rev = srcrev
181 if uri.startswith('git://'):
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500182 __run('git fetch')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500183 __run('git checkout %s' % rev)
184 __run('git tag -f devtool-base-new')
185 md5 = None
186 sha256 = None
Brad Bishop316dfdd2018-06-25 12:45:53 -0400187 _, _, _, _, _, params = bb.fetch2.decodeurl(uri)
188 srcsubdir_rel = params.get('destsuffix', 'git')
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500189 if not srcbranch:
190 check_branch, check_branch_err = __run('git branch -r --contains %s' % srcrev)
191 get_branch = [x.strip() for x in check_branch.splitlines()]
192 # Remove HEAD reference point and drop remote prefix
193 get_branch = [x.split('/', 1)[1] for x in get_branch if not x.startswith('origin/HEAD')]
194 if 'master' in get_branch:
195 # If it is master, we do not need to append 'branch=master' as this is default.
196 # Even with the case where get_branch has multiple objects, if 'master' is one
197 # of them, we should default take from 'master'
198 srcbranch = ''
199 elif len(get_branch) == 1:
200 # If 'master' isn't in get_branch and get_branch contains only ONE object, then store result into 'srcbranch'
201 srcbranch = get_branch[0]
202 else:
203 # If get_branch contains more than one objects, then display error and exit.
204 mbrch = '\n ' + '\n '.join(get_branch)
205 raise DevtoolError('Revision %s was found on multiple branches: %s\nPlease provide the correct branch in the devtool command with "--srcbranch" or "-B" option.' % (srcrev, mbrch))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500206 else:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500207 __run('git checkout devtool-base -b devtool-%s' % newpv)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500208
209 tmpdir = tempfile.mkdtemp(prefix='devtool')
210 try:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500211 checksums, ftmpdir = scriptutils.fetch_url(tinfoil, uri, rev, tmpdir, logger, preserve_tmp=keep_temp)
212 except scriptutils.FetchUrlFailure as e:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500213 raise DevtoolError(e)
214
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500215 if ftmpdir and keep_temp:
216 logger.info('Fetch temp directory is %s' % ftmpdir)
217
218 md5 = checksums['md5sum']
219 sha256 = checksums['sha256sum']
220
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500221 tmpsrctree = _get_srctree(tmpdir)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500222 srctree = os.path.abspath(srctree)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400223 srcsubdir_rel = os.path.relpath(tmpsrctree, tmpdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500224
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500225 # Delete all sources so we ensure no stray files are left over
226 for item in os.listdir(srctree):
227 if item in ['.git', 'oe-local-files']:
228 continue
229 itempath = os.path.join(srctree, item)
230 if os.path.isdir(itempath):
231 shutil.rmtree(itempath)
232 else:
233 os.remove(itempath)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500234
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500235 # Copy in new ones
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500236 _copy_source_code(tmpsrctree, srctree)
237
238 (stdout,_) = __run('git ls-files --modified --others --exclude-standard')
Brad Bishop316dfdd2018-06-25 12:45:53 -0400239 filelist = stdout.splitlines()
240 pbar = bb.ui.knotty.BBProgress('Adding changed files', len(filelist))
241 pbar.start()
242 batchsize = 100
243 for i in range(0, len(filelist), batchsize):
244 batch = filelist[i:i+batchsize]
245 __run('git add -A %s' % ' '.join(['"%s"' % item for item in batch]))
246 pbar.update(i)
247 pbar.finish()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500248
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600249 useroptions = []
250 oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=rd)
251 __run('git %s commit -q -m "Commit of upstream changes at version %s" --allow-empty' % (' '.join(useroptions), newpv))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500252 __run('git tag -f devtool-base-%s' % newpv)
253
254 (stdout, _) = __run('git rev-parse HEAD')
255 rev = stdout.rstrip()
256
257 if no_patch:
258 patches = oe.recipeutils.get_recipe_patches(crd)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400259 if patches:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800260 logger.warning('By user choice, the following patches will NOT be applied to the new source tree:\n %s' % '\n '.join([os.path.basename(patch) for patch in patches]))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500261 else:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600262 __run('git checkout devtool-patched -b %s' % branch)
263 skiptag = False
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500264 try:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500265 __run('git rebase %s' % rev)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600266 except bb.process.ExecutionError as e:
267 skiptag = True
268 if 'conflict' in e.stdout:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800269 logger.warning('Command \'%s\' failed:\n%s\n\nYou will need to resolve conflicts in order to complete the upgrade.' % (e.command, e.stdout.rstrip()))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600270 else:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800271 logger.warning('Command \'%s\' failed:\n%s' % (e.command, e.stdout))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600272 if not skiptag:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500273 if uri.startswith('git://'):
274 suffix = 'new'
275 else:
276 suffix = newpv
277 __run('git tag -f devtool-patched-%s' % suffix)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500278
279 if tmpsrctree:
280 if keep_temp:
281 logger.info('Preserving temporary directory %s' % tmpsrctree)
282 else:
283 shutil.rmtree(tmpsrctree)
Brad Bishop96ff1982019-08-19 13:50:42 -0400284 shutil.rmtree(tmpdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500285
Brad Bishop316dfdd2018-06-25 12:45:53 -0400286 return (rev, md5, sha256, srcbranch, srcsubdir_rel)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500287
Brad Bishop316dfdd2018-06-25 12:45:53 -0400288def _add_license_diff_to_recipe(path, diff):
289 notice_text = """# FIXME: the LIC_FILES_CHKSUM values have been updated by 'devtool upgrade'.
290# The following is the difference between the old and the new license text.
291# Please update the LICENSE value if needed, and summarize the changes in
292# the commit message via 'License-Update:' tag.
293# (example: 'License-Update: copyright years updated.')
294#
295# The changes:
296#
297"""
298 commented_diff = "\n".join(["# {}".format(l) for l in diff.split('\n')])
299 with open(path, 'rb') as f:
300 orig_content = f.read()
301 with open(path, 'wb') as f:
302 f.write(notice_text.encode())
303 f.write(commented_diff.encode())
304 f.write("\n#\n\n".encode())
305 f.write(orig_content)
306
Brad Bishop96ff1982019-08-19 13:50:42 -0400307def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, srcsubdir_old, srcsubdir_new, workspace, tinfoil, rd, license_diff, new_licenses, srctree, keep_failure):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500308 """Creates the new recipe under workspace"""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500309
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500310 bpn = rd.getVar('BPN')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500311 path = os.path.join(workspace, 'recipes', bpn)
312 bb.utils.mkdirhier(path)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500313 copied, _ = oe.recipeutils.copy_recipe_files(rd, path, all_variants=True)
314 if not copied:
315 raise DevtoolError('Internal error - no files were copied for recipe %s' % bpn)
316 logger.debug('Copied %s to %s' % (copied, path))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500317
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500318 oldpv = rd.getVar('PV')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500319 if not newpv:
320 newpv = oldpv
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500321 origpath = rd.getVar('FILE')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500322 fullpath = _rename_recipe_files(origpath, bpn, oldpv, newpv, path)
323 logger.debug('Upgraded %s => %s' % (origpath, fullpath))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500324
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500325 newvalues = {}
326 if _recipe_contains(rd, 'PV') and newpv != oldpv:
327 newvalues['PV'] = newpv
328
329 if srcrev:
330 newvalues['SRCREV'] = srcrev
331
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500332 if srcbranch:
333 src_uri = oe.recipeutils.split_var_value(rd.getVar('SRC_URI', False) or '')
334 changed = False
335 replacing = True
336 new_src_uri = []
337 for entry in src_uri:
338 scheme, network, path, user, passwd, params = bb.fetch2.decodeurl(entry)
339 if replacing and scheme in ['git', 'gitsm']:
340 branch = params.get('branch', 'master')
341 if rd.expand(branch) != srcbranch:
342 # Handle case where branch is set through a variable
343 res = re.match(r'\$\{([^}@]+)\}', branch)
344 if res:
345 newvalues[res.group(1)] = srcbranch
346 # We know we won't change SRC_URI now, so break out
347 break
348 else:
349 params['branch'] = srcbranch
350 entry = bb.fetch2.encodeurl((scheme, network, path, user, passwd, params))
351 changed = True
352 replacing = False
353 new_src_uri.append(entry)
354 if changed:
355 newvalues['SRC_URI'] = ' '.join(new_src_uri)
356
357 newvalues['PR'] = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500358
Brad Bishop316dfdd2018-06-25 12:45:53 -0400359 # Work out which SRC_URI entries have changed in case the entry uses a name
360 crd = rd.createCopy()
361 crd.setVar('PV', newpv)
362 for var, value in newvalues.items():
363 crd.setVar(var, value)
364 old_src_uri = (rd.getVar('SRC_URI') or '').split()
365 new_src_uri = (crd.getVar('SRC_URI') or '').split()
366 newnames = []
367 addnames = []
368 for newentry in new_src_uri:
369 _, _, _, _, _, params = bb.fetch2.decodeurl(newentry)
370 if 'name' in params:
371 newnames.append(params['name'])
372 if newentry not in old_src_uri:
373 addnames.append(params['name'])
374 # Find what's been set in the original recipe
375 oldnames = []
376 noname = False
377 for varflag in rd.getVarFlags('SRC_URI'):
378 if varflag.endswith(('.md5sum', '.sha256sum')):
379 name = varflag.rsplit('.', 1)[0]
380 if name not in oldnames:
381 oldnames.append(name)
382 elif varflag in ['md5sum', 'sha256sum']:
383 noname = True
384 # Even if SRC_URI has named entries it doesn't have to actually use the name
385 if noname and addnames and addnames[0] not in oldnames:
386 addnames = []
387 # Drop any old names (the name actually might include ${PV})
388 for name in oldnames:
389 if name not in newnames:
390 newvalues['SRC_URI[%s.md5sum]' % name] = None
391 newvalues['SRC_URI[%s.sha256sum]' % name] = None
392
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500393 if md5 and sha256:
Brad Bishop316dfdd2018-06-25 12:45:53 -0400394 if addnames:
395 nameprefix = '%s.' % addnames[0]
396 else:
397 nameprefix = ''
398 newvalues['SRC_URI[%smd5sum]' % nameprefix] = md5
399 newvalues['SRC_URI[%ssha256sum]' % nameprefix] = sha256
400
401 if srcsubdir_new != srcsubdir_old:
402 s_subdir_old = os.path.relpath(os.path.abspath(rd.getVar('S')), rd.getVar('WORKDIR'))
403 s_subdir_new = os.path.relpath(os.path.abspath(crd.getVar('S')), crd.getVar('WORKDIR'))
404 if srcsubdir_old == s_subdir_old and srcsubdir_new != s_subdir_new:
405 # Subdir for old extracted source matches what S points to (it should!)
406 # but subdir for new extracted source doesn't match what S will be
407 newvalues['S'] = '${WORKDIR}/%s' % srcsubdir_new.replace(newpv, '${PV}')
408 if crd.expand(newvalues['S']) == crd.expand('${WORKDIR}/${BP}'):
409 # It's the default, drop it
410 # FIXME what if S is being set in a .inc?
411 newvalues['S'] = None
412 logger.info('Source subdirectory has changed, dropping S value since it now matches the default ("${WORKDIR}/${BP}")')
413 else:
414 logger.info('Source subdirectory has changed, updating S value')
415
416 if license_diff:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800417 newlicchksum = " ".join(["file://{}".format(l['path']) +
418 (";beginline={}".format(l['beginline']) if l['beginline'] else "") +
419 (";endline={}".format(l['endline']) if l['endline'] else "") +
420 (";md5={}".format(l['actual_md5'])) for l in new_licenses])
Brad Bishop316dfdd2018-06-25 12:45:53 -0400421 newvalues["LIC_FILES_CHKSUM"] = newlicchksum
422 _add_license_diff_to_recipe(fullpath, license_diff)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500423
Brad Bishop96ff1982019-08-19 13:50:42 -0400424 try:
425 rd = tinfoil.parse_recipe_file(fullpath, False)
426 except bb.tinfoil.TinfoilCommandFailed as e:
427 _upgrade_error(e, fullpath, srctree, keep_failure, 'Parsing of upgraded recipe failed')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500428 oe.recipeutils.patch_recipe(rd, fullpath, newvalues)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500429
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600430 return fullpath, copied
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500431
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500432
433def _check_git_config():
434 def getconfig(name):
435 try:
436 value = bb.process.run('git config --global %s' % name)[0].strip()
437 except bb.process.ExecutionError as e:
438 if e.exitcode == 1:
439 value = None
440 else:
441 raise
442 return value
443
444 username = getconfig('user.name')
445 useremail = getconfig('user.email')
446 configerr = []
447 if not username:
448 configerr.append('Please set your name using:\n git config --global user.name')
449 if not useremail:
450 configerr.append('Please set your email using:\n git config --global user.email')
451 if configerr:
452 raise DevtoolError('Your git configuration is incomplete which will prevent rebases from working:\n' + '\n'.join(configerr))
453
Brad Bishop316dfdd2018-06-25 12:45:53 -0400454def _extract_licenses(srcpath, recipe_licenses):
455 licenses = []
456 for url in recipe_licenses.split():
457 license = {}
458 (type, host, path, user, pswd, parm) = bb.fetch.decodeurl(url)
459 license['path'] = path
460 license['md5'] = parm.get('md5', '')
461 license['beginline'], license['endline'] = 0, 0
462 if 'beginline' in parm:
463 license['beginline'] = int(parm['beginline'])
464 if 'endline' in parm:
465 license['endline'] = int(parm['endline'])
466 license['text'] = []
467 with open(os.path.join(srcpath, path), 'rb') as f:
468 import hashlib
469 actual_md5 = hashlib.md5()
470 lineno = 0
471 for line in f:
472 lineno += 1
473 if (lineno >= license['beginline']) and ((lineno <= license['endline']) or not license['endline']):
474 license['text'].append(line.decode(errors='ignore'))
475 actual_md5.update(line)
476 license['actual_md5'] = actual_md5.hexdigest()
477 licenses.append(license)
478 return licenses
479
480def _generate_license_diff(old_licenses, new_licenses):
481 need_diff = False
482 for l in new_licenses:
483 if l['md5'] != l['actual_md5']:
484 need_diff = True
485 break
486 if need_diff == False:
487 return None
488
489 import difflib
490 diff = ''
491 for old, new in zip(old_licenses, new_licenses):
492 for line in difflib.unified_diff(old['text'], new['text'], old['path'], new['path']):
493 diff = diff + line
494 return diff
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500495
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500496def upgrade(args, config, basepath, workspace):
497 """Entry point for the devtool 'upgrade' subcommand"""
498
499 if args.recipename in workspace:
500 raise DevtoolError("recipe %s is already in your workspace" % args.recipename)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500501 if args.srcbranch and not args.srcrev:
502 raise DevtoolError("If you specify --srcbranch/-B then you must use --srcrev/-S to specify the revision" % args.recipename)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500503
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500504 _check_git_config()
505
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500506 tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500507 try:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600508 rd = parse_recipe(config, tinfoil, args.recipename, True)
509 if not rd:
510 return 1
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500511
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500512 pn = rd.getVar('PN')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600513 if pn != args.recipename:
514 logger.info('Mapping %s to %s' % (args.recipename, pn))
515 if pn in workspace:
516 raise DevtoolError("recipe %s is already in your workspace" % pn)
517
518 if args.srctree:
519 srctree = os.path.abspath(args.srctree)
520 else:
521 srctree = standard.get_default_srctree(config, pn)
522
Brad Bishop316dfdd2018-06-25 12:45:53 -0400523 # try to automatically discover latest version and revision if not provided on command line
524 if not args.version and not args.srcrev:
525 version_info = oe.recipeutils.get_recipe_upstream_version(rd)
526 if version_info['version'] and not version_info['version'].endswith("new-commits-available"):
527 args.version = version_info['version']
528 if version_info['revision']:
529 args.srcrev = version_info['revision']
530 if not args.version and not args.srcrev:
531 raise DevtoolError("Automatic discovery of latest version/revision failed - you must provide a version using the --version/-V option, or for recipes that fetch from an SCM such as git, the --srcrev/-S option.")
532
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600533 standard._check_compatible_recipe(pn, rd)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500534 old_srcrev = rd.getVar('SRCREV')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600535 if old_srcrev == 'INVALID':
536 old_srcrev = None
537 if old_srcrev and not args.srcrev:
538 raise DevtoolError("Recipe specifies a SRCREV value; you must specify a new one when upgrading")
Brad Bishop316dfdd2018-06-25 12:45:53 -0400539 old_ver = rd.getVar('PV')
540 if old_ver == args.version and old_srcrev == args.srcrev:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600541 raise DevtoolError("Current and upgrade versions are the same version")
Brad Bishop316dfdd2018-06-25 12:45:53 -0400542 if args.version:
543 if bb.utils.vercmp_string(args.version, old_ver) < 0:
544 logger.warning('Upgrade version %s compares as less than the current version %s. If you are using a package feed for on-target upgrades or providing this recipe for general consumption, then you should increment PE in the recipe (or if there is no current PE value set, set it to "1")' % (args.version, old_ver))
545 check_prerelease_version(args.version, 'devtool upgrade')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600546
547 rf = None
Brad Bishop316dfdd2018-06-25 12:45:53 -0400548 license_diff = None
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600549 try:
Brad Bishop316dfdd2018-06-25 12:45:53 -0400550 logger.info('Extracting current version source...')
551 rev1, srcsubdir1 = standard._extract_source(srctree, False, 'devtool-orig', False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides)
552 old_licenses = _extract_licenses(srctree, rd.getVar('LIC_FILES_CHKSUM'))
553 logger.info('Extracting upgraded version source...')
554 rev2, md5, sha256, srcbranch, srcsubdir2 = _extract_new_source(args.version, srctree, args.no_patch,
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500555 args.srcrev, args.srcbranch, args.branch, args.keep_temp,
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600556 tinfoil, rd)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400557 new_licenses = _extract_licenses(srctree, rd.getVar('LIC_FILES_CHKSUM'))
558 license_diff = _generate_license_diff(old_licenses, new_licenses)
Brad Bishop96ff1982019-08-19 13:50:42 -0400559 rf, copied = _create_new_recipe(args.version, md5, sha256, args.srcrev, srcbranch, srcsubdir1, srcsubdir2, config.workspace_path, tinfoil, rd, license_diff, new_licenses, srctree, args.keep_failure)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600560 except bb.process.CmdError as e:
Brad Bishop96ff1982019-08-19 13:50:42 -0400561 _upgrade_error(e, rf, srctree, args.keep_failure)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600562 except DevtoolError as e:
Brad Bishop96ff1982019-08-19 13:50:42 -0400563 _upgrade_error(e, rf, srctree, args.keep_failure)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600564 standard._add_md5(config, pn, os.path.dirname(rf))
565
566 af = _write_append(rf, srctree, args.same_dir, args.no_same_dir, rev2,
567 copied, config.workspace_path, rd)
568 standard._add_md5(config, pn, af)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500569
Brad Bishop316dfdd2018-06-25 12:45:53 -0400570 update_unlockedsigs(basepath, workspace, args.fixed_setup, [pn])
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500571
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600572 logger.info('Upgraded source extracted to %s' % srctree)
573 logger.info('New recipe is %s' % rf)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400574 if license_diff:
575 logger.info('License checksums have been updated in the new recipe; please refer to it for the difference between the old and the new license texts.')
576 finally:
577 tinfoil.shutdown()
578 return 0
579
580def latest_version(args, config, basepath, workspace):
581 """Entry point for the devtool 'latest_version' subcommand"""
582 tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
583 try:
584 rd = parse_recipe(config, tinfoil, args.recipename, True)
585 if not rd:
586 return 1
587 version_info = oe.recipeutils.get_recipe_upstream_version(rd)
588 # "new-commits-available" is an indication that upstream never issues version tags
589 if not version_info['version'].endswith("new-commits-available"):
590 logger.info("Current version: {}".format(version_info['current_version']))
591 logger.info("Latest version: {}".format(version_info['version']))
592 if version_info['revision']:
593 logger.info("Latest version's commit: {}".format(version_info['revision']))
594 else:
595 logger.info("Latest commit: {}".format(version_info['revision']))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600596 finally:
597 tinfoil.shutdown()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500598 return 0
599
Brad Bishop19323692019-04-05 15:28:33 -0400600def check_upgrade_status(args, config, basepath, workspace):
601 if not args.recipe:
602 logger.info("Checking the upstream status for all recipes may take a few minutes")
603 results = oe.recipeutils.get_recipe_upgrade_status(args.recipe)
604 for result in results:
605 # pn, update_status, current, latest, maintainer, latest_commit, no_update_reason
606 if args.all or result[1] != 'MATCH':
607 logger.info("{:25} {:15} {:15} {} {} {}".format( result[0],
608 result[2],
609 result[1] if result[1] != 'UPDATE' else (result[3] if not result[3].endswith("new-commits-available") else "new commits"),
610 result[4],
611 result[5] if result[5] != 'N/A' else "",
612 "cannot be updated due to: %s" %(result[6]) if result[6] else ""))
613
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500614def register_commands(subparsers, context):
615 """Register devtool subcommands from this plugin"""
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500616
617 defsrctree = standard.get_default_srctree(context.config)
618
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500619 parser_upgrade = subparsers.add_parser('upgrade', help='Upgrade an existing recipe',
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500620 description='Upgrades an existing recipe to a new upstream version. Puts the upgraded recipe file into the workspace along with any associated files, and extracts the source tree to a specified location (in case patches need rebasing or adding to as a result of the upgrade).',
621 group='starting')
622 parser_upgrade.add_argument('recipename', help='Name of recipe to upgrade (just name - no version, path or extension)')
623 parser_upgrade.add_argument('srctree', nargs='?', help='Path to where to extract the source tree. If not specified, a subdirectory of %s will be used.' % defsrctree)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400624 parser_upgrade.add_argument('--version', '-V', help='Version to upgrade to (PV). If omitted, latest upstream version will be determined and used, if possible.')
625 parser_upgrade.add_argument('--srcrev', '-S', help='Source revision to upgrade to (useful when fetching from an SCM such as git)')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500626 parser_upgrade.add_argument('--srcbranch', '-B', help='Branch in source repository containing the revision to use (if fetching from an SCM such as git)')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500627 parser_upgrade.add_argument('--branch', '-b', default="devtool", help='Name for new development branch to checkout (default "%(default)s")')
628 parser_upgrade.add_argument('--no-patch', action="store_true", help='Do not apply patches from the recipe to the new source code')
Brad Bishop316dfdd2018-06-25 12:45:53 -0400629 parser_upgrade.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500630 group = parser_upgrade.add_mutually_exclusive_group()
631 group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true")
632 group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true")
633 parser_upgrade.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
Brad Bishop96ff1982019-08-19 13:50:42 -0400634 parser_upgrade.add_argument('--keep-failure', action="store_true", help='Keep failed upgrade recipe and associated files (for debugging)')
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500635 parser_upgrade.set_defaults(func=upgrade, fixed_setup=context.fixed_setup)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400636
637 parser_latest_version = subparsers.add_parser('latest-version', help='Report the latest version of an existing recipe',
638 description='Queries the upstream server for what the latest upstream release is (for git, tags are checked, for tarballs, a list of them is obtained, and one with the highest version number is reported)',
639 group='info')
640 parser_latest_version.add_argument('recipename', help='Name of recipe to query (just name - no version, path or extension)')
641 parser_latest_version.set_defaults(func=latest_version)
Brad Bishop19323692019-04-05 15:28:33 -0400642
643 parser_check_upgrade_status = subparsers.add_parser('check-upgrade-status', help="Report upgradability for multiple (or all) recipes",
644 description="Prints a table of recipes together with versions currently provided by recipes, and latest upstream versions, when there is a later version available",
645 group='info')
646 parser_check_upgrade_status.add_argument('recipe', help='Name of the recipe to report (omit to report upgrade info for all recipes)', nargs='*')
647 parser_check_upgrade_status.add_argument('--all', '-a', help='Show all recipes, not just recipes needing upgrade', action="store_true")
648 parser_check_upgrade_status.set_defaults(func=check_upgrade_status)