Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2 | |
| 3 | # OpenEmbedded Development tool |
| 4 | # |
| 5 | # Copyright (C) 2014-2015 Intel Corporation |
| 6 | # |
Brad Bishop | c342db3 | 2019-05-15 21:57:59 -0400 | [diff] [blame] | 7 | # SPDX-License-Identifier: GPL-2.0-only |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 8 | # |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 9 | |
| 10 | import sys |
| 11 | import os |
| 12 | import argparse |
| 13 | import glob |
| 14 | import re |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 15 | import configparser |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 16 | import subprocess |
| 17 | import logging |
| 18 | |
| 19 | basepath = '' |
| 20 | workspace = {} |
| 21 | config = None |
| 22 | context = None |
| 23 | |
| 24 | |
| 25 | scripts_path = os.path.dirname(os.path.realpath(__file__)) |
| 26 | lib_path = scripts_path + '/lib' |
| 27 | sys.path = sys.path + [lib_path] |
| 28 | from devtool import DevtoolError, setup_tinfoil |
| 29 | import scriptutils |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 30 | import argparse_oe |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 31 | logger = scriptutils.logger_create('devtool') |
| 32 | |
| 33 | plugins = [] |
| 34 | |
| 35 | |
| 36 | class ConfigHandler(object): |
| 37 | config_file = '' |
| 38 | config_obj = None |
| 39 | init_path = '' |
| 40 | workspace_path = '' |
| 41 | |
| 42 | def __init__(self, filename): |
| 43 | self.config_file = filename |
Brad Bishop | 1a4b7ee | 2018-12-16 17:11:34 -0800 | [diff] [blame] | 44 | self.config_obj = configparser.ConfigParser() |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 45 | |
| 46 | def get(self, section, option, default=None): |
| 47 | try: |
| 48 | ret = self.config_obj.get(section, option) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 49 | except (configparser.NoOptionError, configparser.NoSectionError): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 50 | if default != None: |
| 51 | ret = default |
| 52 | else: |
| 53 | raise |
| 54 | return ret |
| 55 | |
| 56 | def read(self): |
| 57 | if os.path.exists(self.config_file): |
| 58 | self.config_obj.read(self.config_file) |
| 59 | |
| 60 | if self.config_obj.has_option('General', 'init_path'): |
| 61 | pth = self.get('General', 'init_path') |
| 62 | self.init_path = os.path.join(basepath, pth) |
| 63 | if not os.path.exists(self.init_path): |
| 64 | logger.error('init_path %s specified in config file cannot be found' % pth) |
| 65 | return False |
| 66 | else: |
| 67 | self.config_obj.add_section('General') |
| 68 | |
| 69 | self.workspace_path = self.get('General', 'workspace_path', os.path.join(basepath, 'workspace')) |
| 70 | return True |
| 71 | |
| 72 | |
| 73 | def write(self): |
| 74 | logger.debug('writing to config file %s' % self.config_file) |
| 75 | self.config_obj.set('General', 'workspace_path', self.workspace_path) |
| 76 | with open(self.config_file, 'w') as f: |
| 77 | self.config_obj.write(f) |
| 78 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 79 | def set(self, section, option, value): |
| 80 | if not self.config_obj.has_section(section): |
| 81 | self.config_obj.add_section(section) |
| 82 | self.config_obj.set(section, option, value) |
| 83 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 84 | class Context: |
| 85 | def __init__(self, **kwargs): |
| 86 | self.__dict__.update(kwargs) |
| 87 | |
| 88 | |
| 89 | def read_workspace(): |
| 90 | global workspace |
| 91 | workspace = {} |
| 92 | if not os.path.exists(os.path.join(config.workspace_path, 'conf', 'layer.conf')): |
| 93 | if context.fixed_setup: |
| 94 | logger.error("workspace layer not set up") |
| 95 | sys.exit(1) |
| 96 | else: |
| 97 | logger.info('Creating workspace layer in %s' % config.workspace_path) |
| 98 | _create_workspace(config.workspace_path, config, basepath) |
| 99 | if not context.fixed_setup: |
| 100 | _enable_workspace_layer(config.workspace_path, config, basepath) |
| 101 | |
| 102 | logger.debug('Reading workspace in %s' % config.workspace_path) |
Patrick Williams | 213cb26 | 2021-08-07 19:21:33 -0500 | [diff] [blame] | 103 | externalsrc_re = re.compile(r'^EXTERNALSRC(:pn-([^ =]+))? *= *"([^"]*)"$') |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 104 | for fn in glob.glob(os.path.join(config.workspace_path, 'appends', '*.bbappend')): |
| 105 | with open(fn, 'r') as f: |
Brad Bishop | 316dfdd | 2018-06-25 12:45:53 -0400 | [diff] [blame] | 106 | pnvalues = {} |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 107 | for line in f: |
| 108 | res = externalsrc_re.match(line.rstrip()) |
| 109 | if res: |
Brad Bishop | 316dfdd | 2018-06-25 12:45:53 -0400 | [diff] [blame] | 110 | recipepn = os.path.splitext(os.path.basename(fn))[0].split('_')[0] |
| 111 | pn = res.group(2) or recipepn |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 112 | # Find the recipe file within the workspace, if any |
| 113 | bbfile = os.path.basename(fn).replace('.bbappend', '.bb').replace('%', '*') |
| 114 | recipefile = glob.glob(os.path.join(config.workspace_path, |
| 115 | 'recipes', |
Brad Bishop | 316dfdd | 2018-06-25 12:45:53 -0400 | [diff] [blame] | 116 | recipepn, |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 117 | bbfile)) |
| 118 | if recipefile: |
| 119 | recipefile = recipefile[0] |
Brad Bishop | 316dfdd | 2018-06-25 12:45:53 -0400 | [diff] [blame] | 120 | pnvalues['srctree'] = res.group(3) |
| 121 | pnvalues['bbappend'] = fn |
| 122 | pnvalues['recipefile'] = recipefile |
| 123 | elif line.startswith('# srctreebase: '): |
| 124 | pnvalues['srctreebase'] = line.split(':', 1)[1].strip() |
| 125 | if pnvalues: |
| 126 | if not pnvalues.get('srctreebase', None): |
| 127 | pnvalues['srctreebase'] = pnvalues['srctree'] |
| 128 | logger.debug('Found recipe %s' % pnvalues) |
| 129 | workspace[pn] = pnvalues |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 130 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 131 | def create_workspace(args, config, basepath, workspace): |
| 132 | if args.layerpath: |
| 133 | workspacedir = os.path.abspath(args.layerpath) |
| 134 | else: |
| 135 | workspacedir = os.path.abspath(os.path.join(basepath, 'workspace')) |
| 136 | _create_workspace(workspacedir, config, basepath) |
| 137 | if not args.create_only: |
| 138 | _enable_workspace_layer(workspacedir, config, basepath) |
| 139 | |
| 140 | def _create_workspace(workspacedir, config, basepath): |
| 141 | import bb |
| 142 | |
| 143 | confdir = os.path.join(workspacedir, 'conf') |
| 144 | if os.path.exists(os.path.join(confdir, 'layer.conf')): |
| 145 | logger.info('Specified workspace already set up, leaving as-is') |
| 146 | else: |
| 147 | # Add a config file |
| 148 | bb.utils.mkdirhier(confdir) |
| 149 | with open(os.path.join(confdir, 'layer.conf'), 'w') as f: |
| 150 | f.write('# ### workspace layer auto-generated by devtool ###\n') |
| 151 | f.write('BBPATH =. "$' + '{LAYERDIR}:"\n') |
| 152 | f.write('BBFILES += "$' + '{LAYERDIR}/recipes/*/*.bb \\\n') |
| 153 | f.write(' $' + '{LAYERDIR}/appends/*.bbappend"\n') |
| 154 | f.write('BBFILE_COLLECTIONS += "workspacelayer"\n') |
| 155 | f.write('BBFILE_PATTERN_workspacelayer = "^$' + '{LAYERDIR}/"\n') |
| 156 | f.write('BBFILE_PATTERN_IGNORE_EMPTY_workspacelayer = "1"\n') |
| 157 | f.write('BBFILE_PRIORITY_workspacelayer = "99"\n') |
Brad Bishop | 316dfdd | 2018-06-25 12:45:53 -0400 | [diff] [blame] | 158 | f.write('LAYERSERIES_COMPAT_workspacelayer = "${LAYERSERIES_COMPAT_core}"\n') |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 159 | # Add a README file |
| 160 | with open(os.path.join(workspacedir, 'README'), 'w') as f: |
| 161 | f.write('This layer was created by the OpenEmbedded devtool utility in order to\n') |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 162 | f.write('contain recipes and bbappends that are currently being worked on. The idea\n') |
| 163 | f.write('is that the contents is temporary - once you have finished working on a\n') |
| 164 | f.write('recipe you use the appropriate method to move the files you have been\n') |
| 165 | f.write('working on to a proper layer. In most instances you should use the\n') |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 166 | f.write('devtool utility to manage files within it rather than modifying files\n') |
| 167 | f.write('directly (although recipes added with "devtool add" will often need\n') |
| 168 | f.write('direct modification.)\n') |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 169 | f.write('\nIf you no longer need to use devtool or the workspace layer\'s contents\n') |
| 170 | f.write('you can remove the path to this workspace layer from your conf/bblayers.conf\n') |
| 171 | f.write('file (and then delete the layer, if you wish).\n') |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 172 | f.write('\nNote that by default, if devtool fetches and unpacks source code, it\n') |
| 173 | f.write('will place it in a subdirectory of a "sources" subdirectory of the\n') |
| 174 | f.write('layer. If you prefer it to be elsewhere you can specify the source\n') |
| 175 | f.write('tree path on the command line.\n') |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 176 | |
| 177 | def _enable_workspace_layer(workspacedir, config, basepath): |
| 178 | """Ensure the workspace layer is in bblayers.conf""" |
| 179 | import bb |
| 180 | bblayers_conf = os.path.join(basepath, 'conf', 'bblayers.conf') |
| 181 | if not os.path.exists(bblayers_conf): |
| 182 | logger.error('Unable to find bblayers.conf') |
| 183 | return |
Brad Bishop | 316dfdd | 2018-06-25 12:45:53 -0400 | [diff] [blame] | 184 | if os.path.abspath(workspacedir) != os.path.abspath(config.workspace_path): |
| 185 | removedir = config.workspace_path |
| 186 | else: |
| 187 | removedir = None |
| 188 | _, added = bb.utils.edit_bblayers_conf(bblayers_conf, workspacedir, removedir) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 189 | if added: |
| 190 | logger.info('Enabling workspace layer in bblayers.conf') |
| 191 | if config.workspace_path != workspacedir: |
| 192 | # Update our config to point to the new location |
| 193 | config.workspace_path = workspacedir |
| 194 | config.write() |
| 195 | |
| 196 | |
| 197 | def main(): |
| 198 | global basepath |
| 199 | global config |
| 200 | global context |
| 201 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 202 | if sys.getfilesystemencoding() != "utf-8": |
| 203 | sys.exit("Please use a locale setting which supports utf-8.\nPython can't change the filesystem locale after loading so we need a utf-8 when python starts or things won't work.") |
| 204 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 205 | context = Context(fixed_setup=False) |
| 206 | |
| 207 | # Default basepath |
| 208 | basepath = os.path.dirname(os.path.abspath(__file__)) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 209 | |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 210 | parser = argparse_oe.ArgumentParser(description="OpenEmbedded development tool", |
| 211 | add_help=False, |
| 212 | epilog="Use %(prog)s <subcommand> --help to get help on a specific command") |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 213 | parser.add_argument('--basepath', help='Base directory of SDK / build directory') |
| 214 | parser.add_argument('--bbpath', help='Explicitly specify the BBPATH, rather than getting it from the metadata') |
| 215 | parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true') |
| 216 | parser.add_argument('-q', '--quiet', help='Print only errors', action='store_true') |
| 217 | parser.add_argument('--color', choices=['auto', 'always', 'never'], default='auto', help='Colorize output (where %(metavar)s is %(choices)s)', metavar='COLOR') |
| 218 | |
| 219 | global_args, unparsed_args = parser.parse_known_args() |
| 220 | |
| 221 | # Help is added here rather than via add_help=True, as we don't want it to |
| 222 | # be handled by parse_known_args() |
| 223 | parser.add_argument('-h', '--help', action='help', default=argparse.SUPPRESS, |
| 224 | help='show this help message and exit') |
| 225 | |
| 226 | if global_args.debug: |
| 227 | logger.setLevel(logging.DEBUG) |
| 228 | elif global_args.quiet: |
| 229 | logger.setLevel(logging.ERROR) |
| 230 | |
| 231 | if global_args.basepath: |
| 232 | # Override |
| 233 | basepath = global_args.basepath |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 234 | if os.path.exists(os.path.join(basepath, '.devtoolbase')): |
| 235 | context.fixed_setup = True |
| 236 | else: |
| 237 | pth = basepath |
| 238 | while pth != '' and pth != os.sep: |
| 239 | if os.path.exists(os.path.join(pth, '.devtoolbase')): |
| 240 | context.fixed_setup = True |
| 241 | basepath = pth |
| 242 | break |
| 243 | pth = os.path.dirname(pth) |
| 244 | |
| 245 | if not context.fixed_setup: |
| 246 | basepath = os.environ.get('BUILDDIR') |
| 247 | if not basepath: |
| 248 | logger.error("This script can only be run after initialising the build environment (e.g. by using oe-init-build-env)") |
| 249 | sys.exit(1) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 250 | |
| 251 | logger.debug('Using basepath %s' % basepath) |
| 252 | |
| 253 | config = ConfigHandler(os.path.join(basepath, 'conf', 'devtool.conf')) |
| 254 | if not config.read(): |
| 255 | return -1 |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 256 | context.config = config |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 257 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 258 | bitbake_subdir = config.get('General', 'bitbake_subdir', '') |
| 259 | if bitbake_subdir: |
| 260 | # Normally set for use within the SDK |
| 261 | logger.debug('Using bitbake subdir %s' % bitbake_subdir) |
| 262 | sys.path.insert(0, os.path.join(basepath, bitbake_subdir, 'lib')) |
| 263 | core_meta_subdir = config.get('General', 'core_meta_subdir') |
| 264 | sys.path.insert(0, os.path.join(basepath, core_meta_subdir, 'lib')) |
| 265 | else: |
| 266 | # Standard location |
| 267 | import scriptpath |
| 268 | bitbakepath = scriptpath.add_bitbake_lib_path() |
| 269 | if not bitbakepath: |
| 270 | logger.error("Unable to find bitbake by searching parent directory of this script or PATH") |
| 271 | sys.exit(1) |
| 272 | logger.debug('Using standard bitbake path %s' % bitbakepath) |
| 273 | scriptpath.add_oe_lib_path() |
| 274 | |
| 275 | scriptutils.logger_setup_color(logger, global_args.color) |
| 276 | |
| 277 | if global_args.bbpath is None: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 278 | try: |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 279 | tinfoil = setup_tinfoil(config_only=True, basepath=basepath) |
| 280 | try: |
| 281 | global_args.bbpath = tinfoil.config_data.getVar('BBPATH') |
| 282 | finally: |
| 283 | tinfoil.shutdown() |
| 284 | except bb.BBHandledException: |
| 285 | return 2 |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 286 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 287 | # Search BBPATH first to allow layers to override plugins in scripts_path |
| 288 | for path in global_args.bbpath.split(':') + [scripts_path]: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 289 | pluginpath = os.path.join(path, 'lib', 'devtool') |
| 290 | scriptutils.load_plugins(logger, plugins, pluginpath) |
| 291 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 292 | subparsers = parser.add_subparsers(dest="subparser_name", title='subcommands', metavar='<subcommand>') |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 293 | subparsers.required = True |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 294 | |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 295 | subparsers.add_subparser_group('sdk', 'SDK maintenance', -2) |
| 296 | subparsers.add_subparser_group('advanced', 'Advanced', -1) |
| 297 | subparsers.add_subparser_group('starting', 'Beginning work on a recipe', 100) |
| 298 | subparsers.add_subparser_group('info', 'Getting information') |
| 299 | subparsers.add_subparser_group('working', 'Working on a recipe in the workspace') |
| 300 | subparsers.add_subparser_group('testbuild', 'Testing changes on target') |
| 301 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 302 | if not context.fixed_setup: |
| 303 | parser_create_workspace = subparsers.add_parser('create-workspace', |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 304 | help='Set up workspace in an alternative location', |
| 305 | description='Sets up a new workspace. NOTE: other devtool subcommands will create a workspace automatically as needed, so you only need to use %(prog)s if you want to specify where the workspace should be located.', |
| 306 | group='advanced') |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 307 | parser_create_workspace.add_argument('layerpath', nargs='?', help='Path in which the workspace layer should be created') |
| 308 | parser_create_workspace.add_argument('--create-only', action="store_true", help='Only create the workspace layer, do not alter configuration') |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 309 | parser_create_workspace.set_defaults(func=create_workspace, no_workspace=True) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 310 | |
| 311 | for plugin in plugins: |
| 312 | if hasattr(plugin, 'register_commands'): |
| 313 | plugin.register_commands(subparsers, context) |
| 314 | |
| 315 | args = parser.parse_args(unparsed_args, namespace=global_args) |
| 316 | |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 317 | if not getattr(args, 'no_workspace', False): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 318 | read_workspace() |
| 319 | |
| 320 | try: |
| 321 | ret = args.func(args, config, basepath, workspace) |
| 322 | except DevtoolError as err: |
| 323 | if str(err): |
| 324 | logger.error(str(err)) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 325 | ret = err.exitcode |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 326 | except argparse_oe.ArgumentUsageError as ae: |
| 327 | parser.error_subcommand(ae.message, ae.subcommand) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 328 | |
| 329 | return ret |
| 330 | |
| 331 | |
| 332 | if __name__ == "__main__": |
| 333 | try: |
| 334 | ret = main() |
| 335 | except Exception: |
| 336 | ret = 1 |
| 337 | import traceback |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 338 | traceback.print_exc() |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 339 | sys.exit(ret) |