blob: aa575d1c0944cd4e752e554d344b96d7f927ddea [file] [log] [blame]
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001import fnmatch
2import logging
3import os
Brad Bishopd7bf8c12018-02-25 22:55:05 -05004import shutil
Patrick Williamsc0f7c042017-02-23 20:41:17 -06005import sys
Brad Bishopd7bf8c12018-02-25 22:55:05 -05006import tempfile
Patrick Williamsc0f7c042017-02-23 20:41:17 -06007
8import bb.utils
9
10from bblayers.common import LayerPlugin
11
12logger = logging.getLogger('bitbake-layers')
13
14
15def plugin_init(plugins):
16 return ActionPlugin()
17
18
19class ActionPlugin(LayerPlugin):
20 def do_add_layer(self, args):
Brad Bishop316dfdd2018-06-25 12:45:53 -040021 """Add one or more layers to bblayers.conf."""
22 layerdirs = [os.path.abspath(ldir) for ldir in args.layerdir]
Patrick Williamsc0f7c042017-02-23 20:41:17 -060023
Brad Bishop316dfdd2018-06-25 12:45:53 -040024 for layerdir in layerdirs:
25 if not os.path.exists(layerdir):
26 sys.stderr.write("Specified layer directory %s doesn't exist\n" % layerdir)
27 return 1
28
29 layer_conf = os.path.join(layerdir, 'conf', 'layer.conf')
30 if not os.path.exists(layer_conf):
31 sys.stderr.write("Specified layer directory %s doesn't contain a conf/layer.conf file\n" % layerdir)
32 return 1
Patrick Williamsc0f7c042017-02-23 20:41:17 -060033
34 bblayers_conf = os.path.join('conf', 'bblayers.conf')
35 if not os.path.exists(bblayers_conf):
36 sys.stderr.write("Unable to find bblayers.conf\n")
37 return 1
38
Brad Bishopd7bf8c12018-02-25 22:55:05 -050039 # Back up bblayers.conf to tempdir before we add layers
40 tempdir = tempfile.mkdtemp()
41 backup = tempdir + "/bblayers.conf.bak"
42 shutil.copy2(bblayers_conf, backup)
43
44 try:
Brad Bishop316dfdd2018-06-25 12:45:53 -040045 notadded, _ = bb.utils.edit_bblayers_conf(bblayers_conf, layerdirs, None)
Brad Bishopd7bf8c12018-02-25 22:55:05 -050046 if not (args.force or notadded):
47 try:
48 self.tinfoil.parseRecipes()
49 except bb.tinfoil.TinfoilUIException:
50 # Restore the back up copy of bblayers.conf
51 shutil.copy2(backup, bblayers_conf)
52 bb.fatal("Parse failure with the specified layer added")
53 else:
54 for item in notadded:
55 sys.stderr.write("Specified layer %s is already in BBLAYERS\n" % item)
56 finally:
57 # Remove the back up copy of bblayers.conf
58 shutil.rmtree(tempdir)
Patrick Williamsc0f7c042017-02-23 20:41:17 -060059
60 def do_remove_layer(self, args):
Brad Bishop316dfdd2018-06-25 12:45:53 -040061 """Remove one or more layers from bblayers.conf."""
Patrick Williamsc0f7c042017-02-23 20:41:17 -060062 bblayers_conf = os.path.join('conf', 'bblayers.conf')
63 if not os.path.exists(bblayers_conf):
64 sys.stderr.write("Unable to find bblayers.conf\n")
65 return 1
66
Brad Bishop316dfdd2018-06-25 12:45:53 -040067 layerdirs = []
68 for item in args.layerdir:
69 if item.startswith('*'):
70 layerdir = item
71 elif not '/' in item:
72 layerdir = '*/%s' % item
73 else:
74 layerdir = os.path.abspath(item)
75 layerdirs.append(layerdir)
76 (_, notremoved) = bb.utils.edit_bblayers_conf(bblayers_conf, None, layerdirs)
Patrick Williamsc0f7c042017-02-23 20:41:17 -060077 if notremoved:
78 for item in notremoved:
79 sys.stderr.write("No layers matching %s found in BBLAYERS\n" % item)
80 return 1
81
82 def do_flatten(self, args):
83 """flatten layer configuration into a separate output directory.
84
85Takes the specified layers (or all layers in the current layer
86configuration if none are specified) and builds a "flattened" directory
87containing the contents of all layers, with any overlayed recipes removed
88and bbappends appended to the corresponding recipes. Note that some manual
89cleanup may still be necessary afterwards, in particular:
90
91* where non-recipe files (such as patches) are overwritten (the flatten
92 command will show a warning for these)
93* where anything beyond the normal layer setup has been added to
94 layer.conf (only the lowest priority number layer's layer.conf is used)
95* overridden/appended items from bbappends will need to be tidied up
96* when the flattened layers do not have the same directory structure (the
97 flatten command should show a warning when this will cause a problem)
98
99Warning: if you flatten several layers where another layer is intended to
100be used "inbetween" them (in layer priority order) such that recipes /
101bbappends in the layers interact, and then attempt to use the new output
102layer together with that other layer, you may no longer get the same
103build results (as the layer priority order has effectively changed).
104"""
105 if len(args.layer) == 1:
106 logger.error('If you specify layers to flatten you must specify at least two')
107 return 1
108
109 outputdir = args.outputdir
110 if os.path.exists(outputdir) and os.listdir(outputdir):
111 logger.error('Directory %s exists and is non-empty, please clear it out first' % outputdir)
112 return 1
113
114 layers = self.bblayers
115 if len(args.layer) > 2:
116 layernames = args.layer
117 found_layernames = []
118 found_layerdirs = []
119 for layerdir in layers:
120 layername = self.get_layer_name(layerdir)
121 if layername in layernames:
122 found_layerdirs.append(layerdir)
123 found_layernames.append(layername)
124
125 for layername in layernames:
126 if not layername in found_layernames:
127 logger.error('Unable to find layer %s in current configuration, please run "%s show-layers" to list configured layers' % (layername, os.path.basename(sys.argv[0])))
128 return
129 layers = found_layerdirs
130 else:
131 layernames = []
132
133 # Ensure a specified path matches our list of layers
134 def layer_path_match(path):
135 for layerdir in layers:
136 if path.startswith(os.path.join(layerdir, '')):
137 return layerdir
138 return None
139
140 applied_appends = []
141 for layer in layers:
142 overlayed = []
143 for f in self.tinfoil.cooker.collection.overlayed.keys():
144 for of in self.tinfoil.cooker.collection.overlayed[f]:
145 if of.startswith(layer):
146 overlayed.append(of)
147
148 logger.plain('Copying files from %s...' % layer )
149 for root, dirs, files in os.walk(layer):
150 if '.git' in dirs:
151 dirs.remove('.git')
152 if '.hg' in dirs:
153 dirs.remove('.hg')
154
155 for f1 in files:
156 f1full = os.sep.join([root, f1])
157 if f1full in overlayed:
158 logger.plain(' Skipping overlayed file %s' % f1full )
159 else:
160 ext = os.path.splitext(f1)[1]
161 if ext != '.bbappend':
162 fdest = f1full[len(layer):]
163 fdest = os.path.normpath(os.sep.join([outputdir,fdest]))
164 bb.utils.mkdirhier(os.path.dirname(fdest))
165 if os.path.exists(fdest):
166 if f1 == 'layer.conf' and root.endswith('/conf'):
167 logger.plain(' Skipping layer config file %s' % f1full )
168 continue
169 else:
170 logger.warning('Overwriting file %s', fdest)
171 bb.utils.copyfile(f1full, fdest)
172 if ext == '.bb':
173 for append in self.tinfoil.cooker.collection.get_file_appends(f1full):
174 if layer_path_match(append):
175 logger.plain(' Applying append %s to %s' % (append, fdest))
176 self.apply_append(append, fdest)
177 applied_appends.append(append)
178
179 # Take care of when some layers are excluded and yet we have included bbappends for those recipes
180 for b in self.tinfoil.cooker.collection.bbappends:
181 (recipename, appendname) = b
182 if appendname not in applied_appends:
183 first_append = None
184 layer = layer_path_match(appendname)
185 if layer:
186 if first_append:
187 self.apply_append(appendname, first_append)
188 else:
189 fdest = appendname[len(layer):]
190 fdest = os.path.normpath(os.sep.join([outputdir,fdest]))
191 bb.utils.mkdirhier(os.path.dirname(fdest))
192 bb.utils.copyfile(appendname, fdest)
193 first_append = fdest
194
195 # Get the regex for the first layer in our list (which is where the conf/layer.conf file will
196 # have come from)
197 first_regex = None
198 layerdir = layers[0]
199 for layername, pattern, regex, _ in self.tinfoil.cooker.bbfile_config_priorities:
200 if regex.match(os.path.join(layerdir, 'test')):
201 first_regex = regex
202 break
203
204 if first_regex:
205 # Find the BBFILES entries that match (which will have come from this conf/layer.conf file)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500206 bbfiles = str(self.tinfoil.config_data.getVar('BBFILES')).split()
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600207 bbfiles_layer = []
208 for item in bbfiles:
209 if first_regex.match(item):
210 newpath = os.path.join(outputdir, item[len(layerdir)+1:])
211 bbfiles_layer.append(newpath)
212
213 if bbfiles_layer:
214 # Check that all important layer files match BBFILES
215 for root, dirs, files in os.walk(outputdir):
216 for f1 in files:
217 ext = os.path.splitext(f1)[1]
218 if ext in ['.bb', '.bbappend']:
219 f1full = os.sep.join([root, f1])
220 entry_found = False
221 for item in bbfiles_layer:
222 if fnmatch.fnmatch(f1full, item):
223 entry_found = True
224 break
225 if not entry_found:
226 logger.warning("File %s does not match the flattened layer's BBFILES setting, you may need to edit conf/layer.conf or move the file elsewhere" % f1full)
227
228 def get_file_layer(self, filename):
229 layerdir = self.get_file_layerdir(filename)
230 if layerdir:
231 return self.get_layer_name(layerdir)
232 else:
233 return '?'
234
235 def get_file_layerdir(self, filename):
236 layer = bb.utils.get_file_layer(filename, self.tinfoil.config_data)
237 return self.bbfile_collections.get(layer, None)
238
239 def apply_append(self, appendname, recipename):
240 with open(appendname, 'r') as appendfile:
241 with open(recipename, 'a') as recipefile:
242 recipefile.write('\n')
243 recipefile.write('##### bbappended from %s #####\n' % self.get_file_layer(appendname))
244 recipefile.writelines(appendfile.readlines())
245
246 def register_commands(self, sp):
247 parser_add_layer = self.add_command(sp, 'add-layer', self.do_add_layer, parserecipes=False)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400248 parser_add_layer.add_argument('layerdir', nargs='+', help='Layer directory/directories to add')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600249
250 parser_remove_layer = self.add_command(sp, 'remove-layer', self.do_remove_layer, parserecipes=False)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400251 parser_remove_layer.add_argument('layerdir', nargs='+', help='Layer directory/directories to remove (wildcards allowed, enclose in quotes to avoid shell expansion)')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600252 parser_remove_layer.set_defaults(func=self.do_remove_layer)
253
254 parser_flatten = self.add_command(sp, 'flatten', self.do_flatten)
255 parser_flatten.add_argument('layer', nargs='*', help='Optional layer(s) to flatten (otherwise all are flattened)')
256 parser_flatten.add_argument('outputdir', help='Output directory')