blob: fa5b8ef3c77890dc262683a98e7b4fd57cc610ad [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# Development tool - upgrade command plugin
2#
Brad Bishopd7bf8c12018-02-25 22:55:05 -05003# Copyright (C) 2014-2017 Intel Corporation
Patrick Williamsc124f4f2015-09-15 14:41:29 -05004#
Brad Bishopc342db32019-05-15 21:57:59 -04005# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -05006#
7"""Devtool upgrade plugin"""
8
9import os
10import sys
11import re
12import shutil
13import tempfile
14import logging
15import argparse
16import scriptutils
17import errno
18import bb
Brad Bishop6e60e8b2018-02-01 10:27:11 -050019
20devtool_path = os.path.dirname(os.path.realpath(__file__)) + '/../../../meta/lib'
21sys.path = sys.path + [devtool_path]
22
Patrick Williamsc124f4f2015-09-15 14:41:29 -050023import oe.recipeutils
24from devtool import standard
Brad Bishop316dfdd2018-06-25 12:45:53 -040025from devtool import exec_build_env_command, setup_tinfoil, DevtoolError, parse_recipe, use_external_build, update_unlockedsigs, check_prerelease_version
Patrick Williamsc124f4f2015-09-15 14:41:29 -050026
27logger = logging.getLogger('devtool')
28
Patrick Williamsc124f4f2015-09-15 14:41:29 -050029def _run(cmd, cwd=''):
30 logger.debug("Running command %s> %s" % (cwd,cmd))
31 return bb.process.run('%s' % cmd, cwd=cwd)
32
33def _get_srctree(tmpdir):
34 srctree = tmpdir
Brad Bishop6dbb3162019-11-25 09:41:34 -050035 dirs = scriptutils.filter_src_subdirs(tmpdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050036 if len(dirs) == 1:
37 srctree = os.path.join(tmpdir, dirs[0])
Patrick Williams2a254922023-08-11 09:48:11 -050038 else:
39 raise DevtoolError("Cannot determine where the source tree is after unpacking in {}: {}".format(tmpdir,dirs))
Patrick Williamsc124f4f2015-09-15 14:41:29 -050040 return srctree
41
42def _copy_source_code(orig, dest):
43 for path in standard._ls_tree(orig):
44 dest_dir = os.path.join(dest, os.path.dirname(path))
45 bb.utils.mkdirhier(dest_dir)
46 dest_path = os.path.join(dest, path)
Patrick Williamsf1e5d692016-03-30 15:21:19 -050047 shutil.move(os.path.join(orig, path), dest_path)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050048
Patrick Williamsc124f4f2015-09-15 14:41:29 -050049def _remove_patch_dirs(recipefolder):
50 for root, dirs, files in os.walk(recipefolder):
51 for d in dirs:
52 shutil.rmtree(os.path.join(root,d))
53
Patrick Williamsf1e5d692016-03-30 15:21:19 -050054def _recipe_contains(rd, var):
Brad Bishop6e60e8b2018-02-01 10:27:11 -050055 rf = rd.getVar('FILE')
Patrick Williamsf1e5d692016-03-30 15:21:19 -050056 varfiles = oe.recipeutils.get_var_files(rf, [var], rd)
Patrick Williamsc0f7c042017-02-23 20:41:17 -060057 for var, fn in varfiles.items():
Patrick Williamsf1e5d692016-03-30 15:21:19 -050058 if fn and fn.startswith(os.path.dirname(rf) + os.sep):
59 return True
60 return False
Patrick Williamsc124f4f2015-09-15 14:41:29 -050061
62def _rename_recipe_dirs(oldpv, newpv, path):
63 for root, dirs, files in os.walk(path):
Patrick Williamsc0f7c042017-02-23 20:41:17 -060064 # Rename directories with the version in their name
Patrick Williamsc124f4f2015-09-15 14:41:29 -050065 for olddir in dirs:
66 if olddir.find(oldpv) != -1:
67 newdir = olddir.replace(oldpv, newpv)
68 if olddir != newdir:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050069 shutil.move(os.path.join(path, olddir), os.path.join(path, newdir))
Patrick Williamsc0f7c042017-02-23 20:41:17 -060070 # Rename any inc files with the version in their name (unusual, but possible)
71 for oldfile in files:
72 if oldfile.endswith('.inc'):
73 if oldfile.find(oldpv) != -1:
74 newfile = oldfile.replace(oldpv, newpv)
75 if oldfile != newfile:
Andrew Geisslerc926e172021-05-07 16:11:35 -050076 bb.utils.rename(os.path.join(path, oldfile),
77 os.path.join(path, newfile))
Patrick Williamsc124f4f2015-09-15 14:41:29 -050078
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050079def _rename_recipe_file(oldrecipe, bpn, oldpv, newpv, path):
80 oldrecipe = os.path.basename(oldrecipe)
81 if oldrecipe.endswith('_%s.bb' % oldpv):
82 newrecipe = '%s_%s.bb' % (bpn, newpv)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050083 if oldrecipe != newrecipe:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050084 shutil.move(os.path.join(path, oldrecipe), os.path.join(path, newrecipe))
Patrick Williamsc124f4f2015-09-15 14:41:29 -050085 else:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050086 newrecipe = oldrecipe
Patrick Williamsc124f4f2015-09-15 14:41:29 -050087 return os.path.join(path, newrecipe)
88
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050089def _rename_recipe_files(oldrecipe, bpn, oldpv, newpv, path):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050090 _rename_recipe_dirs(oldpv, newpv, path)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050091 return _rename_recipe_file(oldrecipe, bpn, oldpv, newpv, path)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050092
Patrick Williamsda295312023-12-05 16:48:56 -060093def _write_append(rc, srctreebase, srctree, same_dir, no_same_dir, revs, copied, workspace, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050094 """Writes an append file"""
95 if not os.path.exists(rc):
96 raise DevtoolError("bbappend not created because %s does not exist" % rc)
97
98 appendpath = os.path.join(workspace, 'appends')
99 if not os.path.exists(appendpath):
100 bb.utils.mkdirhier(appendpath)
101
102 brf = os.path.basename(os.path.splitext(rc)[0]) # rc basename
103
104 srctree = os.path.abspath(srctree)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500105 pn = d.getVar('PN')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500106 af = os.path.join(appendpath, '%s.bbappend' % brf)
107 with open(af, 'w') as f:
Patrick Williams213cb262021-08-07 19:21:33 -0500108 f.write('FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n\n')
Andrew Geissler517393d2023-01-13 08:55:19 -0600109 # Local files can be modified/tracked in separate subdir under srctree
110 # Mostly useful for packages with S != WORKDIR
111 f.write('FILESPATH:prepend := "%s:"\n' %
112 os.path.join(srctreebase, 'oe-local-files'))
113 f.write('# srctreebase: %s\n' % srctreebase)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500114 f.write('inherit externalsrc\n')
115 f.write(('# NOTE: We use pn- overrides here to avoid affecting'
116 'multiple variants in the case where the recipe uses BBCLASSEXTEND\n'))
Patrick Williams213cb262021-08-07 19:21:33 -0500117 f.write('EXTERNALSRC:pn-%s = "%s"\n' % (pn, srctree))
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500118 b_is_s = use_external_build(same_dir, no_same_dir, d)
119 if b_is_s:
Patrick Williams213cb262021-08-07 19:21:33 -0500120 f.write('EXTERNALSRC_BUILD:pn-%s = "%s"\n' % (pn, srctree))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600121 f.write('\n')
Patrick Williamsda295312023-12-05 16:48:56 -0600122 if revs:
123 for name, rev in revs.items():
124 f.write('# initial_rev %s: %s\n' % (name, rev))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600125 if copied:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500126 f.write('# original_path: %s\n' % os.path.dirname(d.getVar('FILE')))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600127 f.write('# original_files: %s\n' % ' '.join(copied))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500128 return af
129
Patrick Williamsdb4c27e2022-08-05 08:10:29 -0500130def _cleanup_on_error(rd, srctree):
Patrick Williamsdb4c27e2022-08-05 08:10:29 -0500131 if os.path.exists(rd):
132 shutil.rmtree(rd)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500133 srctree = os.path.abspath(srctree)
134 if os.path.exists(srctree):
135 shutil.rmtree(srctree)
136
Patrick Williamsdb4c27e2022-08-05 08:10:29 -0500137def _upgrade_error(e, rd, srctree, keep_failure=False, extramsg=None):
138 if not keep_failure:
139 _cleanup_on_error(rd, srctree)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500140 logger.error(e)
Brad Bishop96ff1982019-08-19 13:50:42 -0400141 if extramsg:
142 logger.error(extramsg)
143 if keep_failure:
144 logger.info('Preserving failed upgrade files (--keep-failure)')
145 sys.exit(1)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500146
147def _get_uri(rd):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500148 srcuris = rd.getVar('SRC_URI').split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500149 if not len(srcuris):
150 raise DevtoolError('SRC_URI not found on recipe')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500151 # Get first non-local entry in SRC_URI - usually by convention it's
152 # the first entry, but not always!
153 srcuri = None
154 for entry in srcuris:
155 if not entry.startswith('file://'):
156 srcuri = entry
157 break
158 if not srcuri:
159 raise DevtoolError('Unable to find non-local entry in SRC_URI')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500160 srcrev = '${AUTOREV}'
161 if '://' in srcuri:
162 # Fetch a URL
163 rev_re = re.compile(';rev=([^;]+)')
164 res = rev_re.search(srcuri)
165 if res:
166 srcrev = res.group(1)
167 srcuri = rev_re.sub('', srcuri)
168 return srcuri, srcrev
169
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500170def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, keep_temp, tinfoil, rd):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500171 """Extract sources of a recipe with a new version"""
172
173 def __run(cmd):
174 """Simple wrapper which calls _run with srctree as cwd"""
175 return _run(cmd, srctree)
176
177 crd = rd.createCopy()
178
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500179 pv = crd.getVar('PV')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500180 crd.setVar('PV', newpv)
181
182 tmpsrctree = None
183 uri, rev = _get_uri(crd)
184 if srcrev:
185 rev = srcrev
Patrick Williamsda295312023-12-05 16:48:56 -0600186 paths = [srctree]
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600187 if uri.startswith('git://') or uri.startswith('gitsm://'):
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500188 __run('git fetch')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500189 __run('git checkout %s' % rev)
190 __run('git tag -f devtool-base-new')
Patrick Williamsda295312023-12-05 16:48:56 -0600191 __run('git submodule update --recursive')
192 __run('git submodule foreach \'git tag -f devtool-base-new\'')
193 (stdout, _) = __run('git submodule --quiet foreach \'echo $sm_path\'')
194 paths += [os.path.join(srctree, p) for p in stdout.splitlines()]
Patrick Williams169d7bc2024-01-05 11:33:25 -0600195 checksums = {}
Brad Bishop316dfdd2018-06-25 12:45:53 -0400196 _, _, _, _, _, params = bb.fetch2.decodeurl(uri)
197 srcsubdir_rel = params.get('destsuffix', 'git')
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500198 if not srcbranch:
199 check_branch, check_branch_err = __run('git branch -r --contains %s' % srcrev)
200 get_branch = [x.strip() for x in check_branch.splitlines()]
201 # Remove HEAD reference point and drop remote prefix
202 get_branch = [x.split('/', 1)[1] for x in get_branch if not x.startswith('origin/HEAD')]
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000203 if len(get_branch) == 1:
204 # If srcrev is on only ONE branch, then use that branch
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500205 srcbranch = get_branch[0]
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000206 elif 'main' in get_branch:
207 # If srcrev is on multiple branches, then choose 'main' if it is one of them
208 srcbranch = 'main'
209 elif 'master' in get_branch:
210 # Otherwise choose 'master' if it is one of the branches
211 srcbranch = 'master'
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500212 else:
213 # If get_branch contains more than one objects, then display error and exit.
214 mbrch = '\n ' + '\n '.join(get_branch)
215 raise DevtoolError('Revision %s was found on multiple branches: %s\nPlease provide the correct branch in the devtool command with "--srcbranch" or "-B" option.' % (srcrev, mbrch))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500216 else:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500217 __run('git checkout devtool-base -b devtool-%s' % newpv)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500218
219 tmpdir = tempfile.mkdtemp(prefix='devtool')
220 try:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500221 checksums, ftmpdir = scriptutils.fetch_url(tinfoil, uri, rev, tmpdir, logger, preserve_tmp=keep_temp)
222 except scriptutils.FetchUrlFailure as e:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500223 raise DevtoolError(e)
224
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500225 if ftmpdir and keep_temp:
226 logger.info('Fetch temp directory is %s' % ftmpdir)
227
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500228 tmpsrctree = _get_srctree(tmpdir)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500229 srctree = os.path.abspath(srctree)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400230 srcsubdir_rel = os.path.relpath(tmpsrctree, tmpdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500231
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500232 # Delete all sources so we ensure no stray files are left over
233 for item in os.listdir(srctree):
234 if item in ['.git', 'oe-local-files']:
235 continue
236 itempath = os.path.join(srctree, item)
237 if os.path.isdir(itempath):
238 shutil.rmtree(itempath)
239 else:
240 os.remove(itempath)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500241
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500242 # Copy in new ones
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500243 _copy_source_code(tmpsrctree, srctree)
244
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500245 (stdout,_) = __run('git ls-files --modified --others')
Brad Bishop316dfdd2018-06-25 12:45:53 -0400246 filelist = stdout.splitlines()
247 pbar = bb.ui.knotty.BBProgress('Adding changed files', len(filelist))
248 pbar.start()
249 batchsize = 100
250 for i in range(0, len(filelist), batchsize):
251 batch = filelist[i:i+batchsize]
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500252 __run('git add -f -A %s' % ' '.join(['"%s"' % item for item in batch]))
Brad Bishop316dfdd2018-06-25 12:45:53 -0400253 pbar.update(i)
254 pbar.finish()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500255
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600256 useroptions = []
257 oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=rd)
258 __run('git %s commit -q -m "Commit of upstream changes at version %s" --allow-empty' % (' '.join(useroptions), newpv))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500259 __run('git tag -f devtool-base-%s' % newpv)
260
Patrick Williamsda295312023-12-05 16:48:56 -0600261 revs = {}
262 for path in paths:
263 (stdout, _) = _run('git rev-parse HEAD', cwd=path)
Patrick Williams73bd93f2024-02-20 08:07:48 -0600264 revs[os.path.relpath(path, srctree)] = stdout.rstrip()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500265
266 if no_patch:
267 patches = oe.recipeutils.get_recipe_patches(crd)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400268 if patches:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800269 logger.warning('By user choice, the following patches will NOT be applied to the new source tree:\n %s' % '\n '.join([os.path.basename(patch) for patch in patches]))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500270 else:
Patrick Williamsda295312023-12-05 16:48:56 -0600271 for path in paths:
272 _run('git checkout devtool-patched -b %s' % branch, cwd=path)
273 (stdout, _) = _run('git branch --list devtool-override-*', cwd=path)
274 branches_to_rebase = [branch] + stdout.split()
Patrick Williams73bd93f2024-02-20 08:07:48 -0600275 target_branch = revs[os.path.relpath(path, srctree)]
276
277 # There is a bug (or feature?) in git rebase where if a commit with
278 # a note is fully rebased away by being part of an old commit, the
279 # note is still attached to the old commit. Avoid this by making
280 # sure all old devtool related commits have a note attached to them
281 # (this assumes git config notes.rewriteMode is set to ignore).
282 (stdout, _) = __run('git rev-list devtool-base..%s' % target_branch)
283 for rev in stdout.splitlines():
284 if not oe.patch.GitApplyTree.getNotes(path, rev):
285 oe.patch.GitApplyTree.addNote(path, rev, "dummy")
286
Patrick Williamsda295312023-12-05 16:48:56 -0600287 for b in branches_to_rebase:
Patrick Williams73bd93f2024-02-20 08:07:48 -0600288 logger.info("Rebasing {} onto {}".format(b, target_branch))
Patrick Williamsda295312023-12-05 16:48:56 -0600289 _run('git checkout %s' % b, cwd=path)
290 try:
Patrick Williams73bd93f2024-02-20 08:07:48 -0600291 _run('git rebase %s' % target_branch, cwd=path)
Patrick Williamsda295312023-12-05 16:48:56 -0600292 except bb.process.ExecutionError as e:
293 if 'conflict' in e.stdout:
294 logger.warning('Command \'%s\' failed:\n%s\n\nYou will need to resolve conflicts in order to complete the upgrade.' % (e.command, e.stdout.rstrip()))
295 _run('git rebase --abort', cwd=path)
296 else:
297 logger.warning('Command \'%s\' failed:\n%s' % (e.command, e.stdout))
Patrick Williams73bd93f2024-02-20 08:07:48 -0600298
299 # Remove any dummy notes added above.
300 (stdout, _) = __run('git rev-list devtool-base..%s' % target_branch)
301 for rev in stdout.splitlines():
302 oe.patch.GitApplyTree.removeNote(path, rev, "dummy")
303
Patrick Williamsda295312023-12-05 16:48:56 -0600304 _run('git checkout %s' % branch, cwd=path)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500305
306 if tmpsrctree:
307 if keep_temp:
308 logger.info('Preserving temporary directory %s' % tmpsrctree)
309 else:
310 shutil.rmtree(tmpsrctree)
Brad Bishop6dbb3162019-11-25 09:41:34 -0500311 if tmpdir != tmpsrctree:
312 shutil.rmtree(tmpdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500313
Patrick Williams169d7bc2024-01-05 11:33:25 -0600314 return (revs, checksums, srcbranch, srcsubdir_rel)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500315
Brad Bishop316dfdd2018-06-25 12:45:53 -0400316def _add_license_diff_to_recipe(path, diff):
317 notice_text = """# FIXME: the LIC_FILES_CHKSUM values have been updated by 'devtool upgrade'.
318# The following is the difference between the old and the new license text.
319# Please update the LICENSE value if needed, and summarize the changes in
320# the commit message via 'License-Update:' tag.
321# (example: 'License-Update: copyright years updated.')
322#
323# The changes:
324#
325"""
326 commented_diff = "\n".join(["# {}".format(l) for l in diff.split('\n')])
327 with open(path, 'rb') as f:
328 orig_content = f.read()
329 with open(path, 'wb') as f:
330 f.write(notice_text.encode())
331 f.write(commented_diff.encode())
332 f.write("\n#\n\n".encode())
333 f.write(orig_content)
334
Patrick Williams169d7bc2024-01-05 11:33:25 -0600335def _create_new_recipe(newpv, checksums, srcrev, srcbranch, srcsubdir_old, srcsubdir_new, workspace, tinfoil, rd, license_diff, new_licenses, srctree, keep_failure):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500336 """Creates the new recipe under workspace"""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500337
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500338 bpn = rd.getVar('BPN')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500339 path = os.path.join(workspace, 'recipes', bpn)
340 bb.utils.mkdirhier(path)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500341 copied, _ = oe.recipeutils.copy_recipe_files(rd, path, all_variants=True)
342 if not copied:
343 raise DevtoolError('Internal error - no files were copied for recipe %s' % bpn)
344 logger.debug('Copied %s to %s' % (copied, path))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500345
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500346 oldpv = rd.getVar('PV')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500347 if not newpv:
348 newpv = oldpv
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500349 origpath = rd.getVar('FILE')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500350 fullpath = _rename_recipe_files(origpath, bpn, oldpv, newpv, path)
351 logger.debug('Upgraded %s => %s' % (origpath, fullpath))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500352
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500353 newvalues = {}
354 if _recipe_contains(rd, 'PV') and newpv != oldpv:
355 newvalues['PV'] = newpv
356
357 if srcrev:
358 newvalues['SRCREV'] = srcrev
359
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500360 if srcbranch:
361 src_uri = oe.recipeutils.split_var_value(rd.getVar('SRC_URI', False) or '')
362 changed = False
363 replacing = True
364 new_src_uri = []
365 for entry in src_uri:
Patrick Williamsdb4c27e2022-08-05 08:10:29 -0500366 try:
367 scheme, network, path, user, passwd, params = bb.fetch2.decodeurl(entry)
368 except bb.fetch2.MalformedUrl as e:
369 raise DevtoolError("Could not decode SRC_URI: {}".format(e))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500370 if replacing and scheme in ['git', 'gitsm']:
371 branch = params.get('branch', 'master')
372 if rd.expand(branch) != srcbranch:
373 # Handle case where branch is set through a variable
374 res = re.match(r'\$\{([^}@]+)\}', branch)
375 if res:
376 newvalues[res.group(1)] = srcbranch
377 # We know we won't change SRC_URI now, so break out
378 break
379 else:
380 params['branch'] = srcbranch
381 entry = bb.fetch2.encodeurl((scheme, network, path, user, passwd, params))
382 changed = True
383 replacing = False
384 new_src_uri.append(entry)
385 if changed:
386 newvalues['SRC_URI'] = ' '.join(new_src_uri)
387
388 newvalues['PR'] = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500389
Brad Bishop316dfdd2018-06-25 12:45:53 -0400390 # Work out which SRC_URI entries have changed in case the entry uses a name
391 crd = rd.createCopy()
392 crd.setVar('PV', newpv)
393 for var, value in newvalues.items():
394 crd.setVar(var, value)
395 old_src_uri = (rd.getVar('SRC_URI') or '').split()
396 new_src_uri = (crd.getVar('SRC_URI') or '').split()
397 newnames = []
398 addnames = []
399 for newentry in new_src_uri:
400 _, _, _, _, _, params = bb.fetch2.decodeurl(newentry)
401 if 'name' in params:
402 newnames.append(params['name'])
403 if newentry not in old_src_uri:
404 addnames.append(params['name'])
405 # Find what's been set in the original recipe
406 oldnames = []
Patrick Williams169d7bc2024-01-05 11:33:25 -0600407 oldsums = []
Brad Bishop316dfdd2018-06-25 12:45:53 -0400408 noname = False
409 for varflag in rd.getVarFlags('SRC_URI'):
Patrick Williams169d7bc2024-01-05 11:33:25 -0600410 for checksum in checksums:
411 if varflag.endswith('.' + checksum):
412 name = varflag.rsplit('.', 1)[0]
413 if name not in oldnames:
414 oldnames.append(name)
415 oldsums.append(checksum)
416 elif varflag == checksum:
417 noname = True
418 oldsums.append(checksum)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400419 # Even if SRC_URI has named entries it doesn't have to actually use the name
420 if noname and addnames and addnames[0] not in oldnames:
421 addnames = []
422 # Drop any old names (the name actually might include ${PV})
423 for name in oldnames:
424 if name not in newnames:
Patrick Williams169d7bc2024-01-05 11:33:25 -0600425 for checksum in oldsums:
426 newvalues['SRC_URI[%s.%s]' % (name, checksum)] = None
Brad Bishop316dfdd2018-06-25 12:45:53 -0400427
Patrick Williams169d7bc2024-01-05 11:33:25 -0600428 nameprefix = '%s.' % addnames[0] if addnames else ''
429
430 # md5sum is deprecated, remove any traces of it. If it was the only old
431 # checksum, then replace it with the default checksums.
432 if 'md5sum' in oldsums:
Andrew Geissler1e34c2d2020-05-29 16:02:59 -0500433 newvalues['SRC_URI[%smd5sum]' % nameprefix] = None
Patrick Williams169d7bc2024-01-05 11:33:25 -0600434 oldsums.remove('md5sum')
435 if not oldsums:
436 oldsums = ["%ssum" % s for s in bb.fetch2.SHOWN_CHECKSUM_LIST]
437
438 for checksum in oldsums:
439 newvalues['SRC_URI[%s%s]' % (nameprefix, checksum)] = checksums[checksum]
Brad Bishop316dfdd2018-06-25 12:45:53 -0400440
441 if srcsubdir_new != srcsubdir_old:
442 s_subdir_old = os.path.relpath(os.path.abspath(rd.getVar('S')), rd.getVar('WORKDIR'))
443 s_subdir_new = os.path.relpath(os.path.abspath(crd.getVar('S')), crd.getVar('WORKDIR'))
444 if srcsubdir_old == s_subdir_old and srcsubdir_new != s_subdir_new:
445 # Subdir for old extracted source matches what S points to (it should!)
446 # but subdir for new extracted source doesn't match what S will be
447 newvalues['S'] = '${WORKDIR}/%s' % srcsubdir_new.replace(newpv, '${PV}')
448 if crd.expand(newvalues['S']) == crd.expand('${WORKDIR}/${BP}'):
449 # It's the default, drop it
450 # FIXME what if S is being set in a .inc?
451 newvalues['S'] = None
452 logger.info('Source subdirectory has changed, dropping S value since it now matches the default ("${WORKDIR}/${BP}")')
453 else:
454 logger.info('Source subdirectory has changed, updating S value')
455
456 if license_diff:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800457 newlicchksum = " ".join(["file://{}".format(l['path']) +
458 (";beginline={}".format(l['beginline']) if l['beginline'] else "") +
459 (";endline={}".format(l['endline']) if l['endline'] else "") +
460 (";md5={}".format(l['actual_md5'])) for l in new_licenses])
Brad Bishop316dfdd2018-06-25 12:45:53 -0400461 newvalues["LIC_FILES_CHKSUM"] = newlicchksum
462 _add_license_diff_to_recipe(fullpath, license_diff)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500463
Andrew Geissler220dafd2023-10-04 10:18:08 -0500464 tinfoil.modified_files()
Brad Bishop96ff1982019-08-19 13:50:42 -0400465 try:
466 rd = tinfoil.parse_recipe_file(fullpath, False)
467 except bb.tinfoil.TinfoilCommandFailed as e:
Patrick Williamsdb4c27e2022-08-05 08:10:29 -0500468 _upgrade_error(e, os.path.dirname(fullpath), srctree, keep_failure, 'Parsing of upgraded recipe failed')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500469 oe.recipeutils.patch_recipe(rd, fullpath, newvalues)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500470
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600471 return fullpath, copied
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500472
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500473
474def _check_git_config():
475 def getconfig(name):
476 try:
Andrew Geissler20137392023-10-12 04:59:14 -0600477 value = bb.process.run('git config %s' % name)[0].strip()
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500478 except bb.process.ExecutionError as e:
479 if e.exitcode == 1:
480 value = None
481 else:
482 raise
483 return value
484
485 username = getconfig('user.name')
486 useremail = getconfig('user.email')
487 configerr = []
488 if not username:
489 configerr.append('Please set your name using:\n git config --global user.name')
490 if not useremail:
491 configerr.append('Please set your email using:\n git config --global user.email')
492 if configerr:
493 raise DevtoolError('Your git configuration is incomplete which will prevent rebases from working:\n' + '\n'.join(configerr))
494
Brad Bishop316dfdd2018-06-25 12:45:53 -0400495def _extract_licenses(srcpath, recipe_licenses):
496 licenses = []
497 for url in recipe_licenses.split():
498 license = {}
499 (type, host, path, user, pswd, parm) = bb.fetch.decodeurl(url)
500 license['path'] = path
501 license['md5'] = parm.get('md5', '')
502 license['beginline'], license['endline'] = 0, 0
503 if 'beginline' in parm:
504 license['beginline'] = int(parm['beginline'])
505 if 'endline' in parm:
506 license['endline'] = int(parm['endline'])
507 license['text'] = []
508 with open(os.path.join(srcpath, path), 'rb') as f:
509 import hashlib
510 actual_md5 = hashlib.md5()
511 lineno = 0
512 for line in f:
513 lineno += 1
514 if (lineno >= license['beginline']) and ((lineno <= license['endline']) or not license['endline']):
515 license['text'].append(line.decode(errors='ignore'))
516 actual_md5.update(line)
517 license['actual_md5'] = actual_md5.hexdigest()
518 licenses.append(license)
519 return licenses
520
521def _generate_license_diff(old_licenses, new_licenses):
522 need_diff = False
523 for l in new_licenses:
524 if l['md5'] != l['actual_md5']:
525 need_diff = True
526 break
527 if need_diff == False:
528 return None
529
530 import difflib
531 diff = ''
532 for old, new in zip(old_licenses, new_licenses):
533 for line in difflib.unified_diff(old['text'], new['text'], old['path'], new['path']):
534 diff = diff + line
535 return diff
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500536
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500537def upgrade(args, config, basepath, workspace):
538 """Entry point for the devtool 'upgrade' subcommand"""
539
540 if args.recipename in workspace:
541 raise DevtoolError("recipe %s is already in your workspace" % args.recipename)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500542 if args.srcbranch and not args.srcrev:
543 raise DevtoolError("If you specify --srcbranch/-B then you must use --srcrev/-S to specify the revision" % args.recipename)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500544
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500545 _check_git_config()
546
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500547 tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500548 try:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600549 rd = parse_recipe(config, tinfoil, args.recipename, True)
550 if not rd:
551 return 1
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500552
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500553 pn = rd.getVar('PN')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600554 if pn != args.recipename:
555 logger.info('Mapping %s to %s' % (args.recipename, pn))
556 if pn in workspace:
557 raise DevtoolError("recipe %s is already in your workspace" % pn)
558
559 if args.srctree:
560 srctree = os.path.abspath(args.srctree)
561 else:
562 srctree = standard.get_default_srctree(config, pn)
563
Andrew Geissler517393d2023-01-13 08:55:19 -0600564 srctree_s = standard.get_real_srctree(srctree, rd.getVar('S'), rd.getVar('WORKDIR'))
Andrew Geissler5f350902021-07-23 13:09:54 -0400565
Brad Bishop316dfdd2018-06-25 12:45:53 -0400566 # try to automatically discover latest version and revision if not provided on command line
567 if not args.version and not args.srcrev:
568 version_info = oe.recipeutils.get_recipe_upstream_version(rd)
569 if version_info['version'] and not version_info['version'].endswith("new-commits-available"):
570 args.version = version_info['version']
571 if version_info['revision']:
572 args.srcrev = version_info['revision']
573 if not args.version and not args.srcrev:
574 raise DevtoolError("Automatic discovery of latest version/revision failed - you must provide a version using the --version/-V option, or for recipes that fetch from an SCM such as git, the --srcrev/-S option.")
575
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600576 standard._check_compatible_recipe(pn, rd)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500577 old_srcrev = rd.getVar('SRCREV')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600578 if old_srcrev == 'INVALID':
579 old_srcrev = None
580 if old_srcrev and not args.srcrev:
581 raise DevtoolError("Recipe specifies a SRCREV value; you must specify a new one when upgrading")
Brad Bishop316dfdd2018-06-25 12:45:53 -0400582 old_ver = rd.getVar('PV')
583 if old_ver == args.version and old_srcrev == args.srcrev:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600584 raise DevtoolError("Current and upgrade versions are the same version")
Brad Bishop316dfdd2018-06-25 12:45:53 -0400585 if args.version:
586 if bb.utils.vercmp_string(args.version, old_ver) < 0:
587 logger.warning('Upgrade version %s compares as less than the current version %s. If you are using a package feed for on-target upgrades or providing this recipe for general consumption, then you should increment PE in the recipe (or if there is no current PE value set, set it to "1")' % (args.version, old_ver))
588 check_prerelease_version(args.version, 'devtool upgrade')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600589
590 rf = None
Brad Bishop316dfdd2018-06-25 12:45:53 -0400591 license_diff = None
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600592 try:
Brad Bishop316dfdd2018-06-25 12:45:53 -0400593 logger.info('Extracting current version source...')
594 rev1, srcsubdir1 = standard._extract_source(srctree, False, 'devtool-orig', False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides)
Andrew Geissler5f350902021-07-23 13:09:54 -0400595 old_licenses = _extract_licenses(srctree_s, (rd.getVar('LIC_FILES_CHKSUM') or ""))
Brad Bishop316dfdd2018-06-25 12:45:53 -0400596 logger.info('Extracting upgraded version source...')
Patrick Williams169d7bc2024-01-05 11:33:25 -0600597 rev2, checksums, srcbranch, srcsubdir2 = _extract_new_source(args.version, srctree, args.no_patch,
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500598 args.srcrev, args.srcbranch, args.branch, args.keep_temp,
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600599 tinfoil, rd)
Andrew Geissler5f350902021-07-23 13:09:54 -0400600 new_licenses = _extract_licenses(srctree_s, (rd.getVar('LIC_FILES_CHKSUM') or ""))
Brad Bishop316dfdd2018-06-25 12:45:53 -0400601 license_diff = _generate_license_diff(old_licenses, new_licenses)
Patrick Williams169d7bc2024-01-05 11:33:25 -0600602 rf, copied = _create_new_recipe(args.version, checksums, args.srcrev, srcbranch, srcsubdir1, srcsubdir2, config.workspace_path, tinfoil, rd, license_diff, new_licenses, srctree, args.keep_failure)
Patrick Williamsdb4c27e2022-08-05 08:10:29 -0500603 except (bb.process.CmdError, DevtoolError) as e:
604 recipedir = os.path.join(config.workspace_path, 'recipes', rd.getVar('BPN'))
605 _upgrade_error(e, recipedir, srctree, args.keep_failure)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600606 standard._add_md5(config, pn, os.path.dirname(rf))
607
Andrew Geissler517393d2023-01-13 08:55:19 -0600608 af = _write_append(rf, srctree, srctree_s, args.same_dir, args.no_same_dir, rev2,
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600609 copied, config.workspace_path, rd)
610 standard._add_md5(config, pn, af)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500611
Brad Bishop316dfdd2018-06-25 12:45:53 -0400612 update_unlockedsigs(basepath, workspace, args.fixed_setup, [pn])
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500613
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600614 logger.info('Upgraded source extracted to %s' % srctree)
615 logger.info('New recipe is %s' % rf)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400616 if license_diff:
617 logger.info('License checksums have been updated in the new recipe; please refer to it for the difference between the old and the new license texts.')
Patrick Williams213cb262021-08-07 19:21:33 -0500618 preferred_version = rd.getVar('PREFERRED_VERSION_%s' % rd.getVar('PN'))
619 if preferred_version:
620 logger.warning('Version is pinned to %s via PREFERRED_VERSION; it may need adjustment to match the new version before any further steps are taken' % preferred_version)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400621 finally:
622 tinfoil.shutdown()
623 return 0
624
625def latest_version(args, config, basepath, workspace):
626 """Entry point for the devtool 'latest_version' subcommand"""
627 tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
628 try:
629 rd = parse_recipe(config, tinfoil, args.recipename, True)
630 if not rd:
631 return 1
632 version_info = oe.recipeutils.get_recipe_upstream_version(rd)
633 # "new-commits-available" is an indication that upstream never issues version tags
634 if not version_info['version'].endswith("new-commits-available"):
635 logger.info("Current version: {}".format(version_info['current_version']))
636 logger.info("Latest version: {}".format(version_info['version']))
637 if version_info['revision']:
638 logger.info("Latest version's commit: {}".format(version_info['revision']))
639 else:
640 logger.info("Latest commit: {}".format(version_info['revision']))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600641 finally:
642 tinfoil.shutdown()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500643 return 0
644
Brad Bishop19323692019-04-05 15:28:33 -0400645def check_upgrade_status(args, config, basepath, workspace):
646 if not args.recipe:
647 logger.info("Checking the upstream status for all recipes may take a few minutes")
648 results = oe.recipeutils.get_recipe_upgrade_status(args.recipe)
649 for result in results:
650 # pn, update_status, current, latest, maintainer, latest_commit, no_update_reason
651 if args.all or result[1] != 'MATCH':
Patrick Williams169d7bc2024-01-05 11:33:25 -0600652 print("{:25} {:15} {:15} {} {} {}".format( result[0],
Brad Bishop19323692019-04-05 15:28:33 -0400653 result[2],
654 result[1] if result[1] != 'UPDATE' else (result[3] if not result[3].endswith("new-commits-available") else "new commits"),
655 result[4],
656 result[5] if result[5] != 'N/A' else "",
657 "cannot be updated due to: %s" %(result[6]) if result[6] else ""))
658
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500659def register_commands(subparsers, context):
660 """Register devtool subcommands from this plugin"""
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500661
662 defsrctree = standard.get_default_srctree(context.config)
663
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500664 parser_upgrade = subparsers.add_parser('upgrade', help='Upgrade an existing recipe',
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500665 description='Upgrades an existing recipe to a new upstream version. Puts the upgraded recipe file into the workspace along with any associated files, and extracts the source tree to a specified location (in case patches need rebasing or adding to as a result of the upgrade).',
666 group='starting')
667 parser_upgrade.add_argument('recipename', help='Name of recipe to upgrade (just name - no version, path or extension)')
668 parser_upgrade.add_argument('srctree', nargs='?', help='Path to where to extract the source tree. If not specified, a subdirectory of %s will be used.' % defsrctree)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400669 parser_upgrade.add_argument('--version', '-V', help='Version to upgrade to (PV). If omitted, latest upstream version will be determined and used, if possible.')
670 parser_upgrade.add_argument('--srcrev', '-S', help='Source revision to upgrade to (useful when fetching from an SCM such as git)')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500671 parser_upgrade.add_argument('--srcbranch', '-B', help='Branch in source repository containing the revision to use (if fetching from an SCM such as git)')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500672 parser_upgrade.add_argument('--branch', '-b', default="devtool", help='Name for new development branch to checkout (default "%(default)s")')
673 parser_upgrade.add_argument('--no-patch', action="store_true", help='Do not apply patches from the recipe to the new source code')
Brad Bishop316dfdd2018-06-25 12:45:53 -0400674 parser_upgrade.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500675 group = parser_upgrade.add_mutually_exclusive_group()
676 group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true")
677 group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true")
678 parser_upgrade.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
Brad Bishop96ff1982019-08-19 13:50:42 -0400679 parser_upgrade.add_argument('--keep-failure', action="store_true", help='Keep failed upgrade recipe and associated files (for debugging)')
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500680 parser_upgrade.set_defaults(func=upgrade, fixed_setup=context.fixed_setup)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400681
682 parser_latest_version = subparsers.add_parser('latest-version', help='Report the latest version of an existing recipe',
683 description='Queries the upstream server for what the latest upstream release is (for git, tags are checked, for tarballs, a list of them is obtained, and one with the highest version number is reported)',
684 group='info')
685 parser_latest_version.add_argument('recipename', help='Name of recipe to query (just name - no version, path or extension)')
686 parser_latest_version.set_defaults(func=latest_version)
Brad Bishop19323692019-04-05 15:28:33 -0400687
688 parser_check_upgrade_status = subparsers.add_parser('check-upgrade-status', help="Report upgradability for multiple (or all) recipes",
689 description="Prints a table of recipes together with versions currently provided by recipes, and latest upstream versions, when there is a later version available",
690 group='info')
691 parser_check_upgrade_status.add_argument('recipe', help='Name of the recipe to report (omit to report upgrade info for all recipes)', nargs='*')
692 parser_check_upgrade_status.add_argument('--all', '-a', help='Show all recipes, not just recipes needing upgrade', action="store_true")
693 parser_check_upgrade_status.set_defaults(func=check_upgrade_status)