blob: 3aae7b93b4b531474c5a28f91499cd565cdf0e9f [file] [log] [blame]
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001#!/usr/bin/env python3
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002
3# OpenEmbedded Development tool
4#
5# Copyright (C) 2014-2015 Intel Corporation
6#
Brad Bishopc342db32019-05-15 21:57:59 -04007# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -05008#
Patrick Williamsc124f4f2015-09-15 14:41:29 -05009
10import sys
11import os
12import argparse
13import glob
14import re
Patrick Williamsc0f7c042017-02-23 20:41:17 -060015import configparser
Patrick Williamsc124f4f2015-09-15 14:41:29 -050016import subprocess
17import logging
18
19basepath = ''
20workspace = {}
21config = None
22context = None
23
24
25scripts_path = os.path.dirname(os.path.realpath(__file__))
26lib_path = scripts_path + '/lib'
27sys.path = sys.path + [lib_path]
28from devtool import DevtoolError, setup_tinfoil
29import scriptutils
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050030import argparse_oe
Patrick Williamsc124f4f2015-09-15 14:41:29 -050031logger = scriptutils.logger_create('devtool')
32
33plugins = []
34
35
36class ConfigHandler(object):
37 config_file = ''
38 config_obj = None
39 init_path = ''
40 workspace_path = ''
41
42 def __init__(self, filename):
43 self.config_file = filename
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080044 self.config_obj = configparser.ConfigParser()
Patrick Williamsc124f4f2015-09-15 14:41:29 -050045
46 def get(self, section, option, default=None):
47 try:
48 ret = self.config_obj.get(section, option)
Patrick Williamsc0f7c042017-02-23 20:41:17 -060049 except (configparser.NoOptionError, configparser.NoSectionError):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050050 if default != None:
51 ret = default
52 else:
53 raise
54 return ret
55
56 def read(self):
57 if os.path.exists(self.config_file):
58 self.config_obj.read(self.config_file)
59
60 if self.config_obj.has_option('General', 'init_path'):
61 pth = self.get('General', 'init_path')
62 self.init_path = os.path.join(basepath, pth)
63 if not os.path.exists(self.init_path):
64 logger.error('init_path %s specified in config file cannot be found' % pth)
65 return False
66 else:
67 self.config_obj.add_section('General')
68
69 self.workspace_path = self.get('General', 'workspace_path', os.path.join(basepath, 'workspace'))
70 return True
71
72
73 def write(self):
74 logger.debug('writing to config file %s' % self.config_file)
75 self.config_obj.set('General', 'workspace_path', self.workspace_path)
76 with open(self.config_file, 'w') as f:
77 self.config_obj.write(f)
78
Patrick Williamsc0f7c042017-02-23 20:41:17 -060079 def set(self, section, option, value):
80 if not self.config_obj.has_section(section):
81 self.config_obj.add_section(section)
82 self.config_obj.set(section, option, value)
83
Patrick Williamsc124f4f2015-09-15 14:41:29 -050084class Context:
85 def __init__(self, **kwargs):
86 self.__dict__.update(kwargs)
87
88
89def read_workspace():
90 global workspace
91 workspace = {}
92 if not os.path.exists(os.path.join(config.workspace_path, 'conf', 'layer.conf')):
93 if context.fixed_setup:
94 logger.error("workspace layer not set up")
95 sys.exit(1)
96 else:
97 logger.info('Creating workspace layer in %s' % config.workspace_path)
98 _create_workspace(config.workspace_path, config, basepath)
99 if not context.fixed_setup:
100 _enable_workspace_layer(config.workspace_path, config, basepath)
101
102 logger.debug('Reading workspace in %s' % config.workspace_path)
Patrick Williams213cb262021-08-07 19:21:33 -0500103 externalsrc_re = re.compile(r'^EXTERNALSRC(:pn-([^ =]+))? *= *"([^"]*)"$')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500104 for fn in glob.glob(os.path.join(config.workspace_path, 'appends', '*.bbappend')):
105 with open(fn, 'r') as f:
Brad Bishop316dfdd2018-06-25 12:45:53 -0400106 pnvalues = {}
Patrick Williamsdb4c27e2022-08-05 08:10:29 -0500107 pn = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500108 for line in f:
109 res = externalsrc_re.match(line.rstrip())
110 if res:
Brad Bishop316dfdd2018-06-25 12:45:53 -0400111 recipepn = os.path.splitext(os.path.basename(fn))[0].split('_')[0]
112 pn = res.group(2) or recipepn
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500113 # Find the recipe file within the workspace, if any
114 bbfile = os.path.basename(fn).replace('.bbappend', '.bb').replace('%', '*')
115 recipefile = glob.glob(os.path.join(config.workspace_path,
116 'recipes',
Brad Bishop316dfdd2018-06-25 12:45:53 -0400117 recipepn,
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500118 bbfile))
119 if recipefile:
120 recipefile = recipefile[0]
Brad Bishop316dfdd2018-06-25 12:45:53 -0400121 pnvalues['srctree'] = res.group(3)
122 pnvalues['bbappend'] = fn
123 pnvalues['recipefile'] = recipefile
124 elif line.startswith('# srctreebase: '):
125 pnvalues['srctreebase'] = line.split(':', 1)[1].strip()
126 if pnvalues:
Patrick Williamsdb4c27e2022-08-05 08:10:29 -0500127 if not pn:
128 raise DevtoolError("Found *.bbappend in %s, but could not determine EXTERNALSRC:pn-*. "
129 "Maybe still using old syntax?" % config.workspace_path)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400130 if not pnvalues.get('srctreebase', None):
131 pnvalues['srctreebase'] = pnvalues['srctree']
132 logger.debug('Found recipe %s' % pnvalues)
133 workspace[pn] = pnvalues
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500134
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500135def create_workspace(args, config, basepath, workspace):
136 if args.layerpath:
137 workspacedir = os.path.abspath(args.layerpath)
138 else:
139 workspacedir = os.path.abspath(os.path.join(basepath, 'workspace'))
Andrew Geissler517393d2023-01-13 08:55:19 -0600140 layerseries = None
141 if args.layerseries:
142 layerseries = args.layerseries
143 _create_workspace(workspacedir, config, basepath, layerseries)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500144 if not args.create_only:
145 _enable_workspace_layer(workspacedir, config, basepath)
146
Andrew Geissler517393d2023-01-13 08:55:19 -0600147def _create_workspace(workspacedir, config, basepath, layerseries=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500148 import bb
149
150 confdir = os.path.join(workspacedir, 'conf')
151 if os.path.exists(os.path.join(confdir, 'layer.conf')):
152 logger.info('Specified workspace already set up, leaving as-is')
153 else:
Andrew Geissler517393d2023-01-13 08:55:19 -0600154 if not layerseries:
155 tinfoil = setup_tinfoil(config_only=True, basepath=basepath)
156 try:
157 layerseries = tinfoil.config_data.getVar('LAYERSERIES_CORENAMES')
158 finally:
159 tinfoil.shutdown()
160
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500161 # Add a config file
162 bb.utils.mkdirhier(confdir)
163 with open(os.path.join(confdir, 'layer.conf'), 'w') as f:
164 f.write('# ### workspace layer auto-generated by devtool ###\n')
165 f.write('BBPATH =. "$' + '{LAYERDIR}:"\n')
166 f.write('BBFILES += "$' + '{LAYERDIR}/recipes/*/*.bb \\\n')
167 f.write(' $' + '{LAYERDIR}/appends/*.bbappend"\n')
168 f.write('BBFILE_COLLECTIONS += "workspacelayer"\n')
169 f.write('BBFILE_PATTERN_workspacelayer = "^$' + '{LAYERDIR}/"\n')
170 f.write('BBFILE_PATTERN_IGNORE_EMPTY_workspacelayer = "1"\n')
171 f.write('BBFILE_PRIORITY_workspacelayer = "99"\n')
Andrew Geissler517393d2023-01-13 08:55:19 -0600172 f.write('LAYERSERIES_COMPAT_workspacelayer = "%s"\n' % layerseries)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500173 # Add a README file
174 with open(os.path.join(workspacedir, 'README'), 'w') as f:
175 f.write('This layer was created by the OpenEmbedded devtool utility in order to\n')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600176 f.write('contain recipes and bbappends that are currently being worked on. The idea\n')
177 f.write('is that the contents is temporary - once you have finished working on a\n')
178 f.write('recipe you use the appropriate method to move the files you have been\n')
179 f.write('working on to a proper layer. In most instances you should use the\n')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500180 f.write('devtool utility to manage files within it rather than modifying files\n')
181 f.write('directly (although recipes added with "devtool add" will often need\n')
182 f.write('direct modification.)\n')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600183 f.write('\nIf you no longer need to use devtool or the workspace layer\'s contents\n')
184 f.write('you can remove the path to this workspace layer from your conf/bblayers.conf\n')
185 f.write('file (and then delete the layer, if you wish).\n')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500186 f.write('\nNote that by default, if devtool fetches and unpacks source code, it\n')
187 f.write('will place it in a subdirectory of a "sources" subdirectory of the\n')
188 f.write('layer. If you prefer it to be elsewhere you can specify the source\n')
189 f.write('tree path on the command line.\n')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500190
191def _enable_workspace_layer(workspacedir, config, basepath):
192 """Ensure the workspace layer is in bblayers.conf"""
193 import bb
194 bblayers_conf = os.path.join(basepath, 'conf', 'bblayers.conf')
195 if not os.path.exists(bblayers_conf):
196 logger.error('Unable to find bblayers.conf')
197 return
Brad Bishop316dfdd2018-06-25 12:45:53 -0400198 if os.path.abspath(workspacedir) != os.path.abspath(config.workspace_path):
199 removedir = config.workspace_path
200 else:
201 removedir = None
202 _, added = bb.utils.edit_bblayers_conf(bblayers_conf, workspacedir, removedir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500203 if added:
204 logger.info('Enabling workspace layer in bblayers.conf')
205 if config.workspace_path != workspacedir:
206 # Update our config to point to the new location
207 config.workspace_path = workspacedir
208 config.write()
209
210
211def main():
212 global basepath
213 global config
214 global context
215
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500216 if sys.getfilesystemencoding() != "utf-8":
217 sys.exit("Please use a locale setting which supports utf-8.\nPython can't change the filesystem locale after loading so we need a utf-8 when python starts or things won't work.")
218
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500219 context = Context(fixed_setup=False)
220
221 # Default basepath
222 basepath = os.path.dirname(os.path.abspath(__file__))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500223
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500224 parser = argparse_oe.ArgumentParser(description="OpenEmbedded development tool",
225 add_help=False,
226 epilog="Use %(prog)s <subcommand> --help to get help on a specific command")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500227 parser.add_argument('--basepath', help='Base directory of SDK / build directory')
228 parser.add_argument('--bbpath', help='Explicitly specify the BBPATH, rather than getting it from the metadata')
229 parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true')
230 parser.add_argument('-q', '--quiet', help='Print only errors', action='store_true')
231 parser.add_argument('--color', choices=['auto', 'always', 'never'], default='auto', help='Colorize output (where %(metavar)s is %(choices)s)', metavar='COLOR')
232
233 global_args, unparsed_args = parser.parse_known_args()
234
235 # Help is added here rather than via add_help=True, as we don't want it to
236 # be handled by parse_known_args()
237 parser.add_argument('-h', '--help', action='help', default=argparse.SUPPRESS,
238 help='show this help message and exit')
239
240 if global_args.debug:
241 logger.setLevel(logging.DEBUG)
242 elif global_args.quiet:
243 logger.setLevel(logging.ERROR)
244
245 if global_args.basepath:
246 # Override
247 basepath = global_args.basepath
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500248 if os.path.exists(os.path.join(basepath, '.devtoolbase')):
249 context.fixed_setup = True
250 else:
251 pth = basepath
252 while pth != '' and pth != os.sep:
253 if os.path.exists(os.path.join(pth, '.devtoolbase')):
254 context.fixed_setup = True
255 basepath = pth
256 break
257 pth = os.path.dirname(pth)
258
259 if not context.fixed_setup:
260 basepath = os.environ.get('BUILDDIR')
261 if not basepath:
262 logger.error("This script can only be run after initialising the build environment (e.g. by using oe-init-build-env)")
263 sys.exit(1)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500264
265 logger.debug('Using basepath %s' % basepath)
266
267 config = ConfigHandler(os.path.join(basepath, 'conf', 'devtool.conf'))
268 if not config.read():
269 return -1
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500270 context.config = config
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500271
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500272 bitbake_subdir = config.get('General', 'bitbake_subdir', '')
273 if bitbake_subdir:
274 # Normally set for use within the SDK
275 logger.debug('Using bitbake subdir %s' % bitbake_subdir)
276 sys.path.insert(0, os.path.join(basepath, bitbake_subdir, 'lib'))
277 core_meta_subdir = config.get('General', 'core_meta_subdir')
278 sys.path.insert(0, os.path.join(basepath, core_meta_subdir, 'lib'))
279 else:
280 # Standard location
281 import scriptpath
282 bitbakepath = scriptpath.add_bitbake_lib_path()
283 if not bitbakepath:
284 logger.error("Unable to find bitbake by searching parent directory of this script or PATH")
285 sys.exit(1)
286 logger.debug('Using standard bitbake path %s' % bitbakepath)
287 scriptpath.add_oe_lib_path()
288
289 scriptutils.logger_setup_color(logger, global_args.color)
290
291 if global_args.bbpath is None:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600292 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500293 tinfoil = setup_tinfoil(config_only=True, basepath=basepath)
294 try:
295 global_args.bbpath = tinfoil.config_data.getVar('BBPATH')
296 finally:
297 tinfoil.shutdown()
298 except bb.BBHandledException:
299 return 2
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500300
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500301 # Search BBPATH first to allow layers to override plugins in scripts_path
302 for path in global_args.bbpath.split(':') + [scripts_path]:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500303 pluginpath = os.path.join(path, 'lib', 'devtool')
304 scriptutils.load_plugins(logger, plugins, pluginpath)
305
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500306 subparsers = parser.add_subparsers(dest="subparser_name", title='subcommands', metavar='<subcommand>')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600307 subparsers.required = True
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500308
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500309 subparsers.add_subparser_group('sdk', 'SDK maintenance', -2)
310 subparsers.add_subparser_group('advanced', 'Advanced', -1)
311 subparsers.add_subparser_group('starting', 'Beginning work on a recipe', 100)
312 subparsers.add_subparser_group('info', 'Getting information')
313 subparsers.add_subparser_group('working', 'Working on a recipe in the workspace')
314 subparsers.add_subparser_group('testbuild', 'Testing changes on target')
315
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500316 if not context.fixed_setup:
317 parser_create_workspace = subparsers.add_parser('create-workspace',
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500318 help='Set up workspace in an alternative location',
319 description='Sets up a new workspace. NOTE: other devtool subcommands will create a workspace automatically as needed, so you only need to use %(prog)s if you want to specify where the workspace should be located.',
320 group='advanced')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500321 parser_create_workspace.add_argument('layerpath', nargs='?', help='Path in which the workspace layer should be created')
Andrew Geissler517393d2023-01-13 08:55:19 -0600322 parser_create_workspace.add_argument('--layerseries', help='Layer series the workspace should be set to be compatible with')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500323 parser_create_workspace.add_argument('--create-only', action="store_true", help='Only create the workspace layer, do not alter configuration')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500324 parser_create_workspace.set_defaults(func=create_workspace, no_workspace=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500325
326 for plugin in plugins:
327 if hasattr(plugin, 'register_commands'):
328 plugin.register_commands(subparsers, context)
329
330 args = parser.parse_args(unparsed_args, namespace=global_args)
331
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500332 try:
Patrick Williamsdb4c27e2022-08-05 08:10:29 -0500333 if not getattr(args, 'no_workspace', False):
334 read_workspace()
335
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500336 ret = args.func(args, config, basepath, workspace)
337 except DevtoolError as err:
338 if str(err):
339 logger.error(str(err))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600340 ret = err.exitcode
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500341 except argparse_oe.ArgumentUsageError as ae:
342 parser.error_subcommand(ae.message, ae.subcommand)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500343
344 return ret
345
346
347if __name__ == "__main__":
348 try:
349 ret = main()
350 except Exception:
351 ret = 1
352 import traceback
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500353 traceback.print_exc()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500354 sys.exit(ret)