blob: d39c474fbdc96a4f80f4a2de0cc20db242cce5f0 [file] [log] [blame]
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001#!/usr/bin/env python3
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002
3# Development tool - utility functions for plugins
4#
5# Copyright (C) 2014 Intel Corporation
6#
Brad Bishopc342db32019-05-15 21:57:59 -04007# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -05008#
Patrick Williamsc124f4f2015-09-15 14:41:29 -05009"""Devtool plugins module"""
10
11import os
12import sys
13import subprocess
14import logging
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050015import re
Brad Bishop6e60e8b2018-02-01 10:27:11 -050016import codecs
Patrick Williamsc124f4f2015-09-15 14:41:29 -050017
18logger = logging.getLogger('devtool')
19
Patrick Williamsc124f4f2015-09-15 14:41:29 -050020class DevtoolError(Exception):
21 """Exception for handling devtool errors"""
Patrick Williamsc0f7c042017-02-23 20:41:17 -060022 def __init__(self, message, exitcode=1):
23 super(DevtoolError, self).__init__(message)
24 self.exitcode = exitcode
Patrick Williamsc124f4f2015-09-15 14:41:29 -050025
26
27def exec_build_env_command(init_path, builddir, cmd, watch=False, **options):
28 """Run a program in bitbake build context"""
29 import bb
30 if not 'cwd' in options:
31 options["cwd"] = builddir
32 if init_path:
33 # As the OE init script makes use of BASH_SOURCE to determine OEROOT,
34 # and can't determine it when running under dash, we need to set
35 # the executable to bash to correctly set things up
36 if not 'executable' in options:
37 options['executable'] = 'bash'
38 logger.debug('Executing command: "%s" using init path %s' % (cmd, init_path))
39 init_prefix = '. %s %s > /dev/null && ' % (init_path, builddir)
40 else:
41 logger.debug('Executing command "%s"' % cmd)
42 init_prefix = ''
43 if watch:
44 if sys.stdout.isatty():
45 # Fool bitbake into thinking it's outputting to a terminal (because it is, indirectly)
46 cmd = 'script -e -q -c "%s" /dev/null' % cmd
47 return exec_watch('%s%s' % (init_prefix, cmd), **options)
48 else:
49 return bb.process.run('%s%s' % (init_prefix, cmd), **options)
50
51def exec_watch(cmd, **options):
52 """Run program with stdout shown on sys.stdout"""
53 import bb
Patrick Williamsc0f7c042017-02-23 20:41:17 -060054 if isinstance(cmd, str) and not "shell" in options:
Patrick Williamsc124f4f2015-09-15 14:41:29 -050055 options["shell"] = True
56
57 process = subprocess.Popen(
58 cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **options
59 )
60
Brad Bishop6e60e8b2018-02-01 10:27:11 -050061 reader = codecs.getreader('utf-8')(process.stdout)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050062 buf = ''
63 while True:
Brad Bishop6e60e8b2018-02-01 10:27:11 -050064 out = reader.read(1, 1)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050065 if out:
66 sys.stdout.write(out)
67 sys.stdout.flush()
68 buf += out
69 elif out == '' and process.poll() != None:
70 break
71
72 if process.returncode != 0:
73 raise bb.process.ExecutionError(cmd, process.returncode, buf, None)
74
75 return buf, None
76
77def exec_fakeroot(d, cmd, **kwargs):
78 """Run a command under fakeroot (pseudo, in fact) so that it picks up the appropriate file permissions"""
79 # Grab the command and check it actually exists
Brad Bishop6e60e8b2018-02-01 10:27:11 -050080 fakerootcmd = d.getVar('FAKEROOTCMD')
Patrick Williamsc124f4f2015-09-15 14:41:29 -050081 if not os.path.exists(fakerootcmd):
82 logger.error('pseudo executable %s could not be found - have you run a build yet? pseudo-native should install this and if you have run any build then that should have been built')
83 return 2
84 # Set up the appropriate environment
85 newenv = dict(os.environ)
Brad Bishop6e60e8b2018-02-01 10:27:11 -050086 fakerootenv = d.getVar('FAKEROOTENV')
Patrick Williamsc124f4f2015-09-15 14:41:29 -050087 for varvalue in fakerootenv.split():
88 if '=' in varvalue:
89 splitval = varvalue.split('=', 1)
90 newenv[splitval[0]] = splitval[1]
91 return subprocess.call("%s %s" % (fakerootcmd, cmd), env=newenv, **kwargs)
92
Patrick Williamsf1e5d692016-03-30 15:21:19 -050093def setup_tinfoil(config_only=False, basepath=None, tracking=False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050094 """Initialize tinfoil api from bitbake"""
95 import scriptpath
Patrick Williamsf1e5d692016-03-30 15:21:19 -050096 orig_cwd = os.path.abspath(os.curdir)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050097 try:
98 if basepath:
99 os.chdir(basepath)
100 bitbakepath = scriptpath.add_bitbake_lib_path()
101 if not bitbakepath:
102 logger.error("Unable to find bitbake by searching parent directory of this script or PATH")
103 sys.exit(1)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500104
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500105 import bb.tinfoil
106 tinfoil = bb.tinfoil.Tinfoil(tracking=tracking)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500107 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500108 tinfoil.logger.setLevel(logger.getEffectiveLevel())
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500109 tinfoil.prepare(config_only)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500110 except bb.tinfoil.TinfoilUIException:
111 tinfoil.shutdown()
112 raise DevtoolError('Failed to start bitbake environment')
113 except:
114 tinfoil.shutdown()
115 raise
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500116 finally:
117 os.chdir(orig_cwd)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500118 return tinfoil
119
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500120def parse_recipe(config, tinfoil, pn, appends, filter_workspace=True):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500121 """Parse the specified recipe"""
122 try:
123 recipefile = tinfoil.get_recipe_file(pn)
124 except bb.providers.NoProvider as e:
125 logger.error(str(e))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500126 return None
127 if appends:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500128 append_files = tinfoil.get_file_appends(recipefile)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500129 if filter_workspace:
130 # Filter out appends from the workspace
131 append_files = [path for path in append_files if
132 not path.startswith(config.workspace_path)]
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500133 else:
134 append_files = None
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500135 try:
136 rd = tinfoil.parse_recipe_file(recipefile, appends, append_files)
137 except Exception as e:
138 logger.error(str(e))
139 return None
140 return rd
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500141
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500142def check_workspace_recipe(workspace, pn, checksrc=True, bbclassextend=False):
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500143 """
144 Check that a recipe is in the workspace and (optionally) that source
145 is present.
146 """
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500147
148 workspacepn = pn
149
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600150 for recipe, value in workspace.items():
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500151 if recipe == pn:
152 break
153 if bbclassextend:
154 recipefile = value['recipefile']
155 if recipefile:
156 targets = get_bbclassextend_targets(recipefile, recipe)
157 if pn in targets:
158 workspacepn = recipe
159 break
160 else:
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500161 raise DevtoolError("No recipe named '%s' in your workspace" % pn)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500162
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500163 if checksrc:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500164 srctree = workspace[workspacepn]['srctree']
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500165 if not os.path.exists(srctree):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500166 raise DevtoolError("Source tree %s for recipe %s does not exist" % (srctree, workspacepn))
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500167 if not os.listdir(srctree):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500168 raise DevtoolError("Source tree %s for recipe %s is empty" % (srctree, workspacepn))
169
170 return workspacepn
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500171
172def use_external_build(same_dir, no_same_dir, d):
173 """
174 Determine if we should use B!=S (separate build and source directories) or not
175 """
176 b_is_s = True
177 if no_same_dir:
178 logger.info('Using separate build directory since --no-same-dir specified')
179 b_is_s = False
180 elif same_dir:
181 logger.info('Using source tree as build directory since --same-dir specified')
182 elif bb.data.inherits_class('autotools-brokensep', d):
183 logger.info('Using source tree as build directory since recipe inherits autotools-brokensep')
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500184 elif os.path.abspath(d.getVar('B')) == os.path.abspath(d.getVar('S')):
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500185 logger.info('Using source tree as build directory since that would be the default for this recipe')
186 else:
187 b_is_s = False
188 return b_is_s
189
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600190def setup_git_repo(repodir, version, devbranch, basetag='devtool-base', d=None):
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500191 """
192 Set up the git repository for the source tree
193 """
194 import bb.process
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600195 import oe.patch
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500196 if not os.path.exists(os.path.join(repodir, '.git')):
197 bb.process.run('git init', cwd=repodir)
Brad Bishop19323692019-04-05 15:28:33 -0400198 bb.process.run('git config --local gc.autodetach 0', cwd=repodir)
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500199 bb.process.run('git add .', cwd=repodir)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600200 commit_cmd = ['git']
201 oe.patch.GitApplyTree.gitCommandUserOptions(commit_cmd, d=d)
202 commit_cmd += ['commit', '-q']
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500203 stdout, _ = bb.process.run('git status --porcelain', cwd=repodir)
204 if not stdout:
205 commit_cmd.append('--allow-empty')
206 commitmsg = "Initial empty commit with no upstream sources"
207 elif version:
208 commitmsg = "Initial commit from upstream at version %s" % version
209 else:
210 commitmsg = "Initial commit from upstream"
211 commit_cmd += ['-m', commitmsg]
212 bb.process.run(commit_cmd, cwd=repodir)
213
Brad Bishop316dfdd2018-06-25 12:45:53 -0400214 # Ensure singletask.lock (as used by externalsrc.bbclass) is ignored by git
215 excludes = []
216 excludefile = os.path.join(repodir, '.git', 'info', 'exclude')
217 try:
218 with open(excludefile, 'r') as f:
219 excludes = f.readlines()
220 except FileNotFoundError:
221 pass
222 if 'singletask.lock\n' not in excludes:
223 excludes.append('singletask.lock\n')
224 with open(excludefile, 'w') as f:
225 for line in excludes:
226 f.write(line)
227
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500228 bb.process.run('git checkout -b %s' % devbranch, cwd=repodir)
229 bb.process.run('git tag -f %s' % basetag, cwd=repodir)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500230
231def recipe_to_append(recipefile, config, wildcard=False):
232 """
233 Convert a recipe file to a bbappend file path within the workspace.
234 NOTE: if the bbappend already exists, you should be using
235 workspace[args.recipename]['bbappend'] instead of calling this
236 function.
237 """
238 appendname = os.path.splitext(os.path.basename(recipefile))[0]
239 if wildcard:
240 appendname = re.sub(r'_.*', '_%', appendname)
241 appendpath = os.path.join(config.workspace_path, 'appends')
242 appendfile = os.path.join(appendpath, appendname + '.bbappend')
243 return appendfile
244
245def get_bbclassextend_targets(recipefile, pn):
246 """
247 Cheap function to get BBCLASSEXTEND and then convert that to the
248 list of targets that would result.
249 """
250 import bb.utils
251
252 values = {}
253 def get_bbclassextend_varfunc(varname, origvalue, op, newlines):
254 values[varname] = origvalue
255 return origvalue, None, 0, True
256 with open(recipefile, 'r') as f:
257 bb.utils.edit_metadata(f, ['BBCLASSEXTEND'], get_bbclassextend_varfunc)
258
259 targets = []
260 bbclassextend = values.get('BBCLASSEXTEND', '').split()
261 if bbclassextend:
262 for variant in bbclassextend:
263 if variant == 'nativesdk':
264 targets.append('%s-%s' % (variant, pn))
265 elif variant in ['native', 'cross', 'crosssdk']:
266 targets.append('%s-%s' % (pn, variant))
267 return targets
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600268
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500269def replace_from_file(path, old, new):
270 """Replace strings on a file"""
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600271
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500272 def read_file(path):
273 data = None
274 with open(path) as f:
275 data = f.read()
276 return data
277
278 def write_file(path, data):
279 if data is None:
280 return
281 wdata = data.rstrip() + "\n"
282 with open(path, "w") as f:
283 f.write(wdata)
284
285 # In case old is None, return immediately
286 if old is None:
287 return
288 try:
289 rdata = read_file(path)
290 except IOError as e:
291 # if file does not exit, just quit, otherwise raise an exception
292 if e.errno == errno.ENOENT:
293 return
294 else:
295 raise
296
297 old_contents = rdata.splitlines()
298 new_contents = []
299 for old_content in old_contents:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600300 try:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500301 new_contents.append(old_content.replace(old, new))
302 except ValueError:
303 pass
304 write_file(path, "\n".join(new_contents))
305
306
307def update_unlockedsigs(basepath, workspace, fixed_setup, extra=None):
308 """ This function will make unlocked-sigs.inc match the recipes in the
309 workspace plus any extras we want unlocked. """
310
311 if not fixed_setup:
312 # Only need to write this out within the eSDK
313 return
314
315 if not extra:
316 extra = []
317
318 confdir = os.path.join(basepath, 'conf')
319 unlockedsigs = os.path.join(confdir, 'unlocked-sigs.inc')
320
321 # Get current unlocked list if any
322 values = {}
323 def get_unlockedsigs_varfunc(varname, origvalue, op, newlines):
324 values[varname] = origvalue
325 return origvalue, None, 0, True
326 if os.path.exists(unlockedsigs):
327 with open(unlockedsigs, 'r') as f:
328 bb.utils.edit_metadata(f, ['SIGGEN_UNLOCKED_RECIPES'], get_unlockedsigs_varfunc)
329 unlocked = sorted(values.get('SIGGEN_UNLOCKED_RECIPES', []))
330
331 # If the new list is different to the current list, write it out
332 newunlocked = sorted(list(workspace.keys()) + extra)
333 if unlocked != newunlocked:
334 bb.utils.mkdirhier(confdir)
335 with open(unlockedsigs, 'w') as f:
336 f.write("# DO NOT MODIFY! YOUR CHANGES WILL BE LOST.\n" +
337 "# This layer was created by the OpenEmbedded devtool" +
338 " utility in order to\n" +
339 "# contain recipes that are unlocked.\n")
340
341 f.write('SIGGEN_UNLOCKED_RECIPES += "\\\n')
342 for pn in newunlocked:
343 f.write(' ' + pn)
344 f.write('"')
Brad Bishop316dfdd2018-06-25 12:45:53 -0400345
346def check_prerelease_version(ver, operation):
347 if 'pre' in ver or 'rc' in ver:
348 logger.warning('Version "%s" looks like a pre-release version. '
349 'If that is the case, in order to ensure that the '
350 'version doesn\'t appear to go backwards when you '
351 'later upgrade to the final release version, it is '
352 'recommmended that instead you use '
353 '<current version>+<pre-release version> e.g. if '
354 'upgrading from 1.9 to 2.0-rc2 use "1.9+2.0-rc2". '
355 'If you prefer not to reset and re-try, you can change '
356 'the version after %s succeeds using "devtool rename" '
357 'with -V/--version.' % (ver, operation))
358
359def check_git_repo_dirty(repodir):
360 """Check if a git repository is clean or not"""
361 stdout, _ = bb.process.run('git status --porcelain', cwd=repodir)
362 return stdout
363
364def check_git_repo_op(srctree, ignoredirs=None):
365 """Check if a git repository is in the middle of a rebase"""
366 stdout, _ = bb.process.run('git rev-parse --show-toplevel', cwd=srctree)
367 topleveldir = stdout.strip()
368 if ignoredirs and topleveldir in ignoredirs:
369 return
370 gitdir = os.path.join(topleveldir, '.git')
371 if os.path.exists(os.path.join(gitdir, 'rebase-merge')):
372 raise DevtoolError("Source tree %s appears to be in the middle of a rebase - please resolve this first" % srctree)
373 if os.path.exists(os.path.join(gitdir, 'rebase-apply')):
374 raise DevtoolError("Source tree %s appears to be in the middle of 'git am' or 'git apply' - please resolve this first" % srctree)