Brad Bishop | c342db3 | 2019-05-15 21:57:59 -0400 | [diff] [blame] | 1 | # |
| 2 | # SPDX-License-Identifier: GPL-2.0-only |
| 3 | # |
| 4 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 5 | import fnmatch |
| 6 | import logging |
| 7 | import os |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 8 | import shutil |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 9 | import sys |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 10 | import tempfile |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 11 | |
| 12 | import bb.utils |
| 13 | |
| 14 | from bblayers.common import LayerPlugin |
| 15 | |
| 16 | logger = logging.getLogger('bitbake-layers') |
| 17 | |
| 18 | |
| 19 | def plugin_init(plugins): |
| 20 | return ActionPlugin() |
| 21 | |
| 22 | |
| 23 | class ActionPlugin(LayerPlugin): |
| 24 | def do_add_layer(self, args): |
Brad Bishop | 316dfdd | 2018-06-25 12:45:53 -0400 | [diff] [blame] | 25 | """Add one or more layers to bblayers.conf.""" |
| 26 | layerdirs = [os.path.abspath(ldir) for ldir in args.layerdir] |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 27 | |
Brad Bishop | 316dfdd | 2018-06-25 12:45:53 -0400 | [diff] [blame] | 28 | for layerdir in layerdirs: |
| 29 | if not os.path.exists(layerdir): |
| 30 | sys.stderr.write("Specified layer directory %s doesn't exist\n" % layerdir) |
| 31 | return 1 |
| 32 | |
| 33 | layer_conf = os.path.join(layerdir, 'conf', 'layer.conf') |
| 34 | if not os.path.exists(layer_conf): |
| 35 | sys.stderr.write("Specified layer directory %s doesn't contain a conf/layer.conf file\n" % layerdir) |
| 36 | return 1 |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 37 | |
| 38 | bblayers_conf = os.path.join('conf', 'bblayers.conf') |
| 39 | if not os.path.exists(bblayers_conf): |
| 40 | sys.stderr.write("Unable to find bblayers.conf\n") |
| 41 | return 1 |
| 42 | |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 43 | # Back up bblayers.conf to tempdir before we add layers |
| 44 | tempdir = tempfile.mkdtemp() |
| 45 | backup = tempdir + "/bblayers.conf.bak" |
| 46 | shutil.copy2(bblayers_conf, backup) |
| 47 | |
| 48 | try: |
Brad Bishop | 316dfdd | 2018-06-25 12:45:53 -0400 | [diff] [blame] | 49 | notadded, _ = bb.utils.edit_bblayers_conf(bblayers_conf, layerdirs, None) |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 50 | if not (args.force or notadded): |
| 51 | try: |
Brad Bishop | 1a4b7ee | 2018-12-16 17:11:34 -0800 | [diff] [blame] | 52 | self.tinfoil.run_command('parseConfiguration') |
Andrew Geissler | d1e8949 | 2021-02-12 15:35:20 -0600 | [diff] [blame] | 53 | except (bb.tinfoil.TinfoilUIException, bb.BBHandledException): |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 54 | # Restore the back up copy of bblayers.conf |
| 55 | shutil.copy2(backup, bblayers_conf) |
Andrew Geissler | 7e0e3c0 | 2022-02-25 20:34:39 +0000 | [diff] [blame] | 56 | bb.fatal("Parse failure with the specified layer added, exiting.") |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 57 | else: |
| 58 | for item in notadded: |
| 59 | sys.stderr.write("Specified layer %s is already in BBLAYERS\n" % item) |
| 60 | finally: |
| 61 | # Remove the back up copy of bblayers.conf |
| 62 | shutil.rmtree(tempdir) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 63 | |
| 64 | def do_remove_layer(self, args): |
Brad Bishop | 316dfdd | 2018-06-25 12:45:53 -0400 | [diff] [blame] | 65 | """Remove one or more layers from bblayers.conf.""" |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 66 | bblayers_conf = os.path.join('conf', 'bblayers.conf') |
| 67 | if not os.path.exists(bblayers_conf): |
| 68 | sys.stderr.write("Unable to find bblayers.conf\n") |
| 69 | return 1 |
| 70 | |
Brad Bishop | 316dfdd | 2018-06-25 12:45:53 -0400 | [diff] [blame] | 71 | layerdirs = [] |
| 72 | for item in args.layerdir: |
| 73 | if item.startswith('*'): |
| 74 | layerdir = item |
| 75 | elif not '/' in item: |
| 76 | layerdir = '*/%s' % item |
| 77 | else: |
| 78 | layerdir = os.path.abspath(item) |
| 79 | layerdirs.append(layerdir) |
| 80 | (_, notremoved) = bb.utils.edit_bblayers_conf(bblayers_conf, None, layerdirs) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 81 | if notremoved: |
| 82 | for item in notremoved: |
| 83 | sys.stderr.write("No layers matching %s found in BBLAYERS\n" % item) |
| 84 | return 1 |
| 85 | |
| 86 | def do_flatten(self, args): |
| 87 | """flatten layer configuration into a separate output directory. |
| 88 | |
| 89 | Takes the specified layers (or all layers in the current layer |
| 90 | configuration if none are specified) and builds a "flattened" directory |
| 91 | containing the contents of all layers, with any overlayed recipes removed |
| 92 | and bbappends appended to the corresponding recipes. Note that some manual |
| 93 | cleanup may still be necessary afterwards, in particular: |
| 94 | |
| 95 | * where non-recipe files (such as patches) are overwritten (the flatten |
| 96 | command will show a warning for these) |
| 97 | * where anything beyond the normal layer setup has been added to |
| 98 | layer.conf (only the lowest priority number layer's layer.conf is used) |
| 99 | * overridden/appended items from bbappends will need to be tidied up |
| 100 | * when the flattened layers do not have the same directory structure (the |
| 101 | flatten command should show a warning when this will cause a problem) |
| 102 | |
| 103 | Warning: if you flatten several layers where another layer is intended to |
| 104 | be used "inbetween" them (in layer priority order) such that recipes / |
| 105 | bbappends in the layers interact, and then attempt to use the new output |
| 106 | layer together with that other layer, you may no longer get the same |
| 107 | build results (as the layer priority order has effectively changed). |
| 108 | """ |
| 109 | if len(args.layer) == 1: |
| 110 | logger.error('If you specify layers to flatten you must specify at least two') |
| 111 | return 1 |
| 112 | |
| 113 | outputdir = args.outputdir |
| 114 | if os.path.exists(outputdir) and os.listdir(outputdir): |
| 115 | logger.error('Directory %s exists and is non-empty, please clear it out first' % outputdir) |
| 116 | return 1 |
| 117 | |
| 118 | layers = self.bblayers |
| 119 | if len(args.layer) > 2: |
| 120 | layernames = args.layer |
| 121 | found_layernames = [] |
| 122 | found_layerdirs = [] |
| 123 | for layerdir in layers: |
| 124 | layername = self.get_layer_name(layerdir) |
| 125 | if layername in layernames: |
| 126 | found_layerdirs.append(layerdir) |
| 127 | found_layernames.append(layername) |
| 128 | |
| 129 | for layername in layernames: |
| 130 | if not layername in found_layernames: |
| 131 | logger.error('Unable to find layer %s in current configuration, please run "%s show-layers" to list configured layers' % (layername, os.path.basename(sys.argv[0]))) |
| 132 | return |
| 133 | layers = found_layerdirs |
| 134 | else: |
| 135 | layernames = [] |
| 136 | |
| 137 | # Ensure a specified path matches our list of layers |
| 138 | def layer_path_match(path): |
| 139 | for layerdir in layers: |
| 140 | if path.startswith(os.path.join(layerdir, '')): |
| 141 | return layerdir |
| 142 | return None |
| 143 | |
| 144 | applied_appends = [] |
| 145 | for layer in layers: |
Andrew Geissler | 5a43b43 | 2020-06-13 10:46:56 -0500 | [diff] [blame] | 146 | overlayed = set() |
| 147 | for mc in self.tinfoil.cooker.multiconfigs: |
| 148 | for f in self.tinfoil.cooker.collections[mc].overlayed.keys(): |
| 149 | for of in self.tinfoil.cooker.collections[mc].overlayed[f]: |
| 150 | if of.startswith(layer): |
| 151 | overlayed.add(of) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 152 | |
| 153 | logger.plain('Copying files from %s...' % layer ) |
| 154 | for root, dirs, files in os.walk(layer): |
| 155 | if '.git' in dirs: |
| 156 | dirs.remove('.git') |
| 157 | if '.hg' in dirs: |
| 158 | dirs.remove('.hg') |
| 159 | |
| 160 | for f1 in files: |
| 161 | f1full = os.sep.join([root, f1]) |
| 162 | if f1full in overlayed: |
| 163 | logger.plain(' Skipping overlayed file %s' % f1full ) |
| 164 | else: |
| 165 | ext = os.path.splitext(f1)[1] |
| 166 | if ext != '.bbappend': |
| 167 | fdest = f1full[len(layer):] |
| 168 | fdest = os.path.normpath(os.sep.join([outputdir,fdest])) |
| 169 | bb.utils.mkdirhier(os.path.dirname(fdest)) |
| 170 | if os.path.exists(fdest): |
| 171 | if f1 == 'layer.conf' and root.endswith('/conf'): |
| 172 | logger.plain(' Skipping layer config file %s' % f1full ) |
| 173 | continue |
| 174 | else: |
| 175 | logger.warning('Overwriting file %s', fdest) |
| 176 | bb.utils.copyfile(f1full, fdest) |
| 177 | if ext == '.bb': |
Andrew Geissler | 5a43b43 | 2020-06-13 10:46:56 -0500 | [diff] [blame] | 178 | appends = set() |
| 179 | for mc in self.tinfoil.cooker.multiconfigs: |
| 180 | appends |= set(self.tinfoil.cooker.collections[mc].get_file_appends(f1full)) |
| 181 | for append in appends: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 182 | if layer_path_match(append): |
| 183 | logger.plain(' Applying append %s to %s' % (append, fdest)) |
| 184 | self.apply_append(append, fdest) |
| 185 | applied_appends.append(append) |
| 186 | |
| 187 | # Take care of when some layers are excluded and yet we have included bbappends for those recipes |
Andrew Geissler | 5a43b43 | 2020-06-13 10:46:56 -0500 | [diff] [blame] | 188 | bbappends = set() |
| 189 | for mc in self.tinfoil.cooker.multiconfigs: |
| 190 | bbappends |= set(self.tinfoil.cooker.collections[mc].bbappends) |
| 191 | |
| 192 | for b in bbappends: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 193 | (recipename, appendname) = b |
| 194 | if appendname not in applied_appends: |
| 195 | first_append = None |
| 196 | layer = layer_path_match(appendname) |
| 197 | if layer: |
| 198 | if first_append: |
| 199 | self.apply_append(appendname, first_append) |
| 200 | else: |
| 201 | fdest = appendname[len(layer):] |
| 202 | fdest = os.path.normpath(os.sep.join([outputdir,fdest])) |
| 203 | bb.utils.mkdirhier(os.path.dirname(fdest)) |
| 204 | bb.utils.copyfile(appendname, fdest) |
| 205 | first_append = fdest |
| 206 | |
| 207 | # Get the regex for the first layer in our list (which is where the conf/layer.conf file will |
| 208 | # have come from) |
| 209 | first_regex = None |
| 210 | layerdir = layers[0] |
| 211 | for layername, pattern, regex, _ in self.tinfoil.cooker.bbfile_config_priorities: |
| 212 | if regex.match(os.path.join(layerdir, 'test')): |
| 213 | first_regex = regex |
| 214 | break |
| 215 | |
| 216 | if first_regex: |
| 217 | # Find the BBFILES entries that match (which will have come from this conf/layer.conf file) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 218 | bbfiles = str(self.tinfoil.config_data.getVar('BBFILES')).split() |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 219 | bbfiles_layer = [] |
| 220 | for item in bbfiles: |
| 221 | if first_regex.match(item): |
| 222 | newpath = os.path.join(outputdir, item[len(layerdir)+1:]) |
| 223 | bbfiles_layer.append(newpath) |
| 224 | |
| 225 | if bbfiles_layer: |
| 226 | # Check that all important layer files match BBFILES |
| 227 | for root, dirs, files in os.walk(outputdir): |
| 228 | for f1 in files: |
| 229 | ext = os.path.splitext(f1)[1] |
| 230 | if ext in ['.bb', '.bbappend']: |
| 231 | f1full = os.sep.join([root, f1]) |
| 232 | entry_found = False |
| 233 | for item in bbfiles_layer: |
| 234 | if fnmatch.fnmatch(f1full, item): |
| 235 | entry_found = True |
| 236 | break |
| 237 | if not entry_found: |
| 238 | logger.warning("File %s does not match the flattened layer's BBFILES setting, you may need to edit conf/layer.conf or move the file elsewhere" % f1full) |
| 239 | |
| 240 | def get_file_layer(self, filename): |
| 241 | layerdir = self.get_file_layerdir(filename) |
| 242 | if layerdir: |
| 243 | return self.get_layer_name(layerdir) |
| 244 | else: |
| 245 | return '?' |
| 246 | |
| 247 | def get_file_layerdir(self, filename): |
| 248 | layer = bb.utils.get_file_layer(filename, self.tinfoil.config_data) |
| 249 | return self.bbfile_collections.get(layer, None) |
| 250 | |
| 251 | def apply_append(self, appendname, recipename): |
| 252 | with open(appendname, 'r') as appendfile: |
| 253 | with open(recipename, 'a') as recipefile: |
| 254 | recipefile.write('\n') |
| 255 | recipefile.write('##### bbappended from %s #####\n' % self.get_file_layer(appendname)) |
| 256 | recipefile.writelines(appendfile.readlines()) |
| 257 | |
| 258 | def register_commands(self, sp): |
| 259 | parser_add_layer = self.add_command(sp, 'add-layer', self.do_add_layer, parserecipes=False) |
Brad Bishop | 316dfdd | 2018-06-25 12:45:53 -0400 | [diff] [blame] | 260 | parser_add_layer.add_argument('layerdir', nargs='+', help='Layer directory/directories to add') |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 261 | |
| 262 | parser_remove_layer = self.add_command(sp, 'remove-layer', self.do_remove_layer, parserecipes=False) |
Brad Bishop | 316dfdd | 2018-06-25 12:45:53 -0400 | [diff] [blame] | 263 | parser_remove_layer.add_argument('layerdir', nargs='+', help='Layer directory/directories to remove (wildcards allowed, enclose in quotes to avoid shell expansion)') |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 264 | parser_remove_layer.set_defaults(func=self.do_remove_layer) |
| 265 | |
| 266 | parser_flatten = self.add_command(sp, 'flatten', self.do_flatten) |
| 267 | parser_flatten.add_argument('layer', nargs='*', help='Optional layer(s) to flatten (otherwise all are flattened)') |
| 268 | parser_flatten.add_argument('outputdir', help='Output directory') |