blob: f05f5d330f1ef48e4ce8628f8f1e822802d1b15f [file] [log] [blame]
Brad Bishopc342db32019-05-15 21:57:59 -04001#
2# SPDX-License-Identifier: GPL-2.0-only
3#
4
Patrick Williamsc0f7c042017-02-23 20:41:17 -06005import fnmatch
6import logging
7import os
Brad Bishopd7bf8c12018-02-25 22:55:05 -05008import shutil
Patrick Williamsc0f7c042017-02-23 20:41:17 -06009import sys
Brad Bishopd7bf8c12018-02-25 22:55:05 -050010import tempfile
Patrick Williamsc0f7c042017-02-23 20:41:17 -060011
12import bb.utils
13
14from bblayers.common import LayerPlugin
15
16logger = logging.getLogger('bitbake-layers')
17
18
19def plugin_init(plugins):
20 return ActionPlugin()
21
22
23class ActionPlugin(LayerPlugin):
24 def do_add_layer(self, args):
Brad Bishop316dfdd2018-06-25 12:45:53 -040025 """Add one or more layers to bblayers.conf."""
26 layerdirs = [os.path.abspath(ldir) for ldir in args.layerdir]
Patrick Williamsc0f7c042017-02-23 20:41:17 -060027
Brad Bishop316dfdd2018-06-25 12:45:53 -040028 for layerdir in layerdirs:
29 if not os.path.exists(layerdir):
30 sys.stderr.write("Specified layer directory %s doesn't exist\n" % layerdir)
31 return 1
32
33 layer_conf = os.path.join(layerdir, 'conf', 'layer.conf')
34 if not os.path.exists(layer_conf):
35 sys.stderr.write("Specified layer directory %s doesn't contain a conf/layer.conf file\n" % layerdir)
36 return 1
Patrick Williamsc0f7c042017-02-23 20:41:17 -060037
38 bblayers_conf = os.path.join('conf', 'bblayers.conf')
39 if not os.path.exists(bblayers_conf):
40 sys.stderr.write("Unable to find bblayers.conf\n")
41 return 1
42
Brad Bishopd7bf8c12018-02-25 22:55:05 -050043 # Back up bblayers.conf to tempdir before we add layers
44 tempdir = tempfile.mkdtemp()
45 backup = tempdir + "/bblayers.conf.bak"
46 shutil.copy2(bblayers_conf, backup)
47
48 try:
Brad Bishop316dfdd2018-06-25 12:45:53 -040049 notadded, _ = bb.utils.edit_bblayers_conf(bblayers_conf, layerdirs, None)
Brad Bishopd7bf8c12018-02-25 22:55:05 -050050 if not (args.force or notadded):
51 try:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080052 self.tinfoil.run_command('parseConfiguration')
Andrew Geisslerd1e89492021-02-12 15:35:20 -060053 except (bb.tinfoil.TinfoilUIException, bb.BBHandledException):
Brad Bishopd7bf8c12018-02-25 22:55:05 -050054 # Restore the back up copy of bblayers.conf
55 shutil.copy2(backup, bblayers_conf)
Andrew Geisslerd1e89492021-02-12 15:35:20 -060056 bb.fatal("Parse failure with the specified layer added, aborting.")
Brad Bishopd7bf8c12018-02-25 22:55:05 -050057 else:
58 for item in notadded:
59 sys.stderr.write("Specified layer %s is already in BBLAYERS\n" % item)
60 finally:
61 # Remove the back up copy of bblayers.conf
62 shutil.rmtree(tempdir)
Patrick Williamsc0f7c042017-02-23 20:41:17 -060063
64 def do_remove_layer(self, args):
Brad Bishop316dfdd2018-06-25 12:45:53 -040065 """Remove one or more layers from bblayers.conf."""
Patrick Williamsc0f7c042017-02-23 20:41:17 -060066 bblayers_conf = os.path.join('conf', 'bblayers.conf')
67 if not os.path.exists(bblayers_conf):
68 sys.stderr.write("Unable to find bblayers.conf\n")
69 return 1
70
Brad Bishop316dfdd2018-06-25 12:45:53 -040071 layerdirs = []
72 for item in args.layerdir:
73 if item.startswith('*'):
74 layerdir = item
75 elif not '/' in item:
76 layerdir = '*/%s' % item
77 else:
78 layerdir = os.path.abspath(item)
79 layerdirs.append(layerdir)
80 (_, notremoved) = bb.utils.edit_bblayers_conf(bblayers_conf, None, layerdirs)
Patrick Williamsc0f7c042017-02-23 20:41:17 -060081 if notremoved:
82 for item in notremoved:
83 sys.stderr.write("No layers matching %s found in BBLAYERS\n" % item)
84 return 1
85
86 def do_flatten(self, args):
87 """flatten layer configuration into a separate output directory.
88
89Takes the specified layers (or all layers in the current layer
90configuration if none are specified) and builds a "flattened" directory
91containing the contents of all layers, with any overlayed recipes removed
92and bbappends appended to the corresponding recipes. Note that some manual
93cleanup may still be necessary afterwards, in particular:
94
95* where non-recipe files (such as patches) are overwritten (the flatten
96 command will show a warning for these)
97* where anything beyond the normal layer setup has been added to
98 layer.conf (only the lowest priority number layer's layer.conf is used)
99* overridden/appended items from bbappends will need to be tidied up
100* when the flattened layers do not have the same directory structure (the
101 flatten command should show a warning when this will cause a problem)
102
103Warning: if you flatten several layers where another layer is intended to
104be used "inbetween" them (in layer priority order) such that recipes /
105bbappends in the layers interact, and then attempt to use the new output
106layer together with that other layer, you may no longer get the same
107build results (as the layer priority order has effectively changed).
108"""
109 if len(args.layer) == 1:
110 logger.error('If you specify layers to flatten you must specify at least two')
111 return 1
112
113 outputdir = args.outputdir
114 if os.path.exists(outputdir) and os.listdir(outputdir):
115 logger.error('Directory %s exists and is non-empty, please clear it out first' % outputdir)
116 return 1
117
118 layers = self.bblayers
119 if len(args.layer) > 2:
120 layernames = args.layer
121 found_layernames = []
122 found_layerdirs = []
123 for layerdir in layers:
124 layername = self.get_layer_name(layerdir)
125 if layername in layernames:
126 found_layerdirs.append(layerdir)
127 found_layernames.append(layername)
128
129 for layername in layernames:
130 if not layername in found_layernames:
131 logger.error('Unable to find layer %s in current configuration, please run "%s show-layers" to list configured layers' % (layername, os.path.basename(sys.argv[0])))
132 return
133 layers = found_layerdirs
134 else:
135 layernames = []
136
137 # Ensure a specified path matches our list of layers
138 def layer_path_match(path):
139 for layerdir in layers:
140 if path.startswith(os.path.join(layerdir, '')):
141 return layerdir
142 return None
143
144 applied_appends = []
145 for layer in layers:
Andrew Geissler5a43b432020-06-13 10:46:56 -0500146 overlayed = set()
147 for mc in self.tinfoil.cooker.multiconfigs:
148 for f in self.tinfoil.cooker.collections[mc].overlayed.keys():
149 for of in self.tinfoil.cooker.collections[mc].overlayed[f]:
150 if of.startswith(layer):
151 overlayed.add(of)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600152
153 logger.plain('Copying files from %s...' % layer )
154 for root, dirs, files in os.walk(layer):
155 if '.git' in dirs:
156 dirs.remove('.git')
157 if '.hg' in dirs:
158 dirs.remove('.hg')
159
160 for f1 in files:
161 f1full = os.sep.join([root, f1])
162 if f1full in overlayed:
163 logger.plain(' Skipping overlayed file %s' % f1full )
164 else:
165 ext = os.path.splitext(f1)[1]
166 if ext != '.bbappend':
167 fdest = f1full[len(layer):]
168 fdest = os.path.normpath(os.sep.join([outputdir,fdest]))
169 bb.utils.mkdirhier(os.path.dirname(fdest))
170 if os.path.exists(fdest):
171 if f1 == 'layer.conf' and root.endswith('/conf'):
172 logger.plain(' Skipping layer config file %s' % f1full )
173 continue
174 else:
175 logger.warning('Overwriting file %s', fdest)
176 bb.utils.copyfile(f1full, fdest)
177 if ext == '.bb':
Andrew Geissler5a43b432020-06-13 10:46:56 -0500178 appends = set()
179 for mc in self.tinfoil.cooker.multiconfigs:
180 appends |= set(self.tinfoil.cooker.collections[mc].get_file_appends(f1full))
181 for append in appends:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600182 if layer_path_match(append):
183 logger.plain(' Applying append %s to %s' % (append, fdest))
184 self.apply_append(append, fdest)
185 applied_appends.append(append)
186
187 # Take care of when some layers are excluded and yet we have included bbappends for those recipes
Andrew Geissler5a43b432020-06-13 10:46:56 -0500188 bbappends = set()
189 for mc in self.tinfoil.cooker.multiconfigs:
190 bbappends |= set(self.tinfoil.cooker.collections[mc].bbappends)
191
192 for b in bbappends:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600193 (recipename, appendname) = b
194 if appendname not in applied_appends:
195 first_append = None
196 layer = layer_path_match(appendname)
197 if layer:
198 if first_append:
199 self.apply_append(appendname, first_append)
200 else:
201 fdest = appendname[len(layer):]
202 fdest = os.path.normpath(os.sep.join([outputdir,fdest]))
203 bb.utils.mkdirhier(os.path.dirname(fdest))
204 bb.utils.copyfile(appendname, fdest)
205 first_append = fdest
206
207 # Get the regex for the first layer in our list (which is where the conf/layer.conf file will
208 # have come from)
209 first_regex = None
210 layerdir = layers[0]
211 for layername, pattern, regex, _ in self.tinfoil.cooker.bbfile_config_priorities:
212 if regex.match(os.path.join(layerdir, 'test')):
213 first_regex = regex
214 break
215
216 if first_regex:
217 # Find the BBFILES entries that match (which will have come from this conf/layer.conf file)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500218 bbfiles = str(self.tinfoil.config_data.getVar('BBFILES')).split()
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600219 bbfiles_layer = []
220 for item in bbfiles:
221 if first_regex.match(item):
222 newpath = os.path.join(outputdir, item[len(layerdir)+1:])
223 bbfiles_layer.append(newpath)
224
225 if bbfiles_layer:
226 # Check that all important layer files match BBFILES
227 for root, dirs, files in os.walk(outputdir):
228 for f1 in files:
229 ext = os.path.splitext(f1)[1]
230 if ext in ['.bb', '.bbappend']:
231 f1full = os.sep.join([root, f1])
232 entry_found = False
233 for item in bbfiles_layer:
234 if fnmatch.fnmatch(f1full, item):
235 entry_found = True
236 break
237 if not entry_found:
238 logger.warning("File %s does not match the flattened layer's BBFILES setting, you may need to edit conf/layer.conf or move the file elsewhere" % f1full)
239
240 def get_file_layer(self, filename):
241 layerdir = self.get_file_layerdir(filename)
242 if layerdir:
243 return self.get_layer_name(layerdir)
244 else:
245 return '?'
246
247 def get_file_layerdir(self, filename):
248 layer = bb.utils.get_file_layer(filename, self.tinfoil.config_data)
249 return self.bbfile_collections.get(layer, None)
250
251 def apply_append(self, appendname, recipename):
252 with open(appendname, 'r') as appendfile:
253 with open(recipename, 'a') as recipefile:
254 recipefile.write('\n')
255 recipefile.write('##### bbappended from %s #####\n' % self.get_file_layer(appendname))
256 recipefile.writelines(appendfile.readlines())
257
258 def register_commands(self, sp):
259 parser_add_layer = self.add_command(sp, 'add-layer', self.do_add_layer, parserecipes=False)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400260 parser_add_layer.add_argument('layerdir', nargs='+', help='Layer directory/directories to add')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600261
262 parser_remove_layer = self.add_command(sp, 'remove-layer', self.do_remove_layer, parserecipes=False)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400263 parser_remove_layer.add_argument('layerdir', nargs='+', help='Layer directory/directories to remove (wildcards allowed, enclose in quotes to avoid shell expansion)')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600264 parser_remove_layer.set_defaults(func=self.do_remove_layer)
265
266 parser_flatten = self.add_command(sp, 'flatten', self.do_flatten)
267 parser_flatten.add_argument('layer', nargs='*', help='Optional layer(s) to flatten (otherwise all are flattened)')
268 parser_flatten.add_argument('outputdir', help='Output directory')