blob: 739ae27b97aa19c09cf8d428ad1371bc925b7287 [file] [log] [blame]
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001import fnmatch
2import logging
3import os
4import sys
5
6import bb.utils
7
8from bblayers.common import LayerPlugin
9
10logger = logging.getLogger('bitbake-layers')
11
12
13def plugin_init(plugins):
14 return ActionPlugin()
15
16
17class ActionPlugin(LayerPlugin):
18 def do_add_layer(self, args):
19 """Add a layer to bblayers.conf."""
20 layerdir = os.path.abspath(args.layerdir)
21 if not os.path.exists(layerdir):
22 sys.stderr.write("Specified layer directory doesn't exist\n")
23 return 1
24
25 layer_conf = os.path.join(layerdir, 'conf', 'layer.conf')
26 if not os.path.exists(layer_conf):
27 sys.stderr.write("Specified layer directory doesn't contain a conf/layer.conf file\n")
28 return 1
29
30 bblayers_conf = os.path.join('conf', 'bblayers.conf')
31 if not os.path.exists(bblayers_conf):
32 sys.stderr.write("Unable to find bblayers.conf\n")
33 return 1
34
35 notadded, _ = bb.utils.edit_bblayers_conf(bblayers_conf, layerdir, None)
36 if notadded:
37 for item in notadded:
38 sys.stderr.write("Specified layer %s is already in BBLAYERS\n" % item)
39
40 def do_remove_layer(self, args):
41 """Remove a layer from bblayers.conf."""
42 bblayers_conf = os.path.join('conf', 'bblayers.conf')
43 if not os.path.exists(bblayers_conf):
44 sys.stderr.write("Unable to find bblayers.conf\n")
45 return 1
46
47 if args.layerdir.startswith('*'):
48 layerdir = args.layerdir
49 elif not '/' in args.layerdir:
50 layerdir = '*/%s' % args.layerdir
51 else:
52 layerdir = os.path.abspath(args.layerdir)
53 (_, notremoved) = bb.utils.edit_bblayers_conf(bblayers_conf, None, layerdir)
54 if notremoved:
55 for item in notremoved:
56 sys.stderr.write("No layers matching %s found in BBLAYERS\n" % item)
57 return 1
58
59 def do_flatten(self, args):
60 """flatten layer configuration into a separate output directory.
61
62Takes the specified layers (or all layers in the current layer
63configuration if none are specified) and builds a "flattened" directory
64containing the contents of all layers, with any overlayed recipes removed
65and bbappends appended to the corresponding recipes. Note that some manual
66cleanup may still be necessary afterwards, in particular:
67
68* where non-recipe files (such as patches) are overwritten (the flatten
69 command will show a warning for these)
70* where anything beyond the normal layer setup has been added to
71 layer.conf (only the lowest priority number layer's layer.conf is used)
72* overridden/appended items from bbappends will need to be tidied up
73* when the flattened layers do not have the same directory structure (the
74 flatten command should show a warning when this will cause a problem)
75
76Warning: if you flatten several layers where another layer is intended to
77be used "inbetween" them (in layer priority order) such that recipes /
78bbappends in the layers interact, and then attempt to use the new output
79layer together with that other layer, you may no longer get the same
80build results (as the layer priority order has effectively changed).
81"""
82 if len(args.layer) == 1:
83 logger.error('If you specify layers to flatten you must specify at least two')
84 return 1
85
86 outputdir = args.outputdir
87 if os.path.exists(outputdir) and os.listdir(outputdir):
88 logger.error('Directory %s exists and is non-empty, please clear it out first' % outputdir)
89 return 1
90
91 layers = self.bblayers
92 if len(args.layer) > 2:
93 layernames = args.layer
94 found_layernames = []
95 found_layerdirs = []
96 for layerdir in layers:
97 layername = self.get_layer_name(layerdir)
98 if layername in layernames:
99 found_layerdirs.append(layerdir)
100 found_layernames.append(layername)
101
102 for layername in layernames:
103 if not layername in found_layernames:
104 logger.error('Unable to find layer %s in current configuration, please run "%s show-layers" to list configured layers' % (layername, os.path.basename(sys.argv[0])))
105 return
106 layers = found_layerdirs
107 else:
108 layernames = []
109
110 # Ensure a specified path matches our list of layers
111 def layer_path_match(path):
112 for layerdir in layers:
113 if path.startswith(os.path.join(layerdir, '')):
114 return layerdir
115 return None
116
117 applied_appends = []
118 for layer in layers:
119 overlayed = []
120 for f in self.tinfoil.cooker.collection.overlayed.keys():
121 for of in self.tinfoil.cooker.collection.overlayed[f]:
122 if of.startswith(layer):
123 overlayed.append(of)
124
125 logger.plain('Copying files from %s...' % layer )
126 for root, dirs, files in os.walk(layer):
127 if '.git' in dirs:
128 dirs.remove('.git')
129 if '.hg' in dirs:
130 dirs.remove('.hg')
131
132 for f1 in files:
133 f1full = os.sep.join([root, f1])
134 if f1full in overlayed:
135 logger.plain(' Skipping overlayed file %s' % f1full )
136 else:
137 ext = os.path.splitext(f1)[1]
138 if ext != '.bbappend':
139 fdest = f1full[len(layer):]
140 fdest = os.path.normpath(os.sep.join([outputdir,fdest]))
141 bb.utils.mkdirhier(os.path.dirname(fdest))
142 if os.path.exists(fdest):
143 if f1 == 'layer.conf' and root.endswith('/conf'):
144 logger.plain(' Skipping layer config file %s' % f1full )
145 continue
146 else:
147 logger.warning('Overwriting file %s', fdest)
148 bb.utils.copyfile(f1full, fdest)
149 if ext == '.bb':
150 for append in self.tinfoil.cooker.collection.get_file_appends(f1full):
151 if layer_path_match(append):
152 logger.plain(' Applying append %s to %s' % (append, fdest))
153 self.apply_append(append, fdest)
154 applied_appends.append(append)
155
156 # Take care of when some layers are excluded and yet we have included bbappends for those recipes
157 for b in self.tinfoil.cooker.collection.bbappends:
158 (recipename, appendname) = b
159 if appendname not in applied_appends:
160 first_append = None
161 layer = layer_path_match(appendname)
162 if layer:
163 if first_append:
164 self.apply_append(appendname, first_append)
165 else:
166 fdest = appendname[len(layer):]
167 fdest = os.path.normpath(os.sep.join([outputdir,fdest]))
168 bb.utils.mkdirhier(os.path.dirname(fdest))
169 bb.utils.copyfile(appendname, fdest)
170 first_append = fdest
171
172 # Get the regex for the first layer in our list (which is where the conf/layer.conf file will
173 # have come from)
174 first_regex = None
175 layerdir = layers[0]
176 for layername, pattern, regex, _ in self.tinfoil.cooker.bbfile_config_priorities:
177 if regex.match(os.path.join(layerdir, 'test')):
178 first_regex = regex
179 break
180
181 if first_regex:
182 # Find the BBFILES entries that match (which will have come from this conf/layer.conf file)
183 bbfiles = str(self.tinfoil.config_data.getVar('BBFILES', True)).split()
184 bbfiles_layer = []
185 for item in bbfiles:
186 if first_regex.match(item):
187 newpath = os.path.join(outputdir, item[len(layerdir)+1:])
188 bbfiles_layer.append(newpath)
189
190 if bbfiles_layer:
191 # Check that all important layer files match BBFILES
192 for root, dirs, files in os.walk(outputdir):
193 for f1 in files:
194 ext = os.path.splitext(f1)[1]
195 if ext in ['.bb', '.bbappend']:
196 f1full = os.sep.join([root, f1])
197 entry_found = False
198 for item in bbfiles_layer:
199 if fnmatch.fnmatch(f1full, item):
200 entry_found = True
201 break
202 if not entry_found:
203 logger.warning("File %s does not match the flattened layer's BBFILES setting, you may need to edit conf/layer.conf or move the file elsewhere" % f1full)
204
205 def get_file_layer(self, filename):
206 layerdir = self.get_file_layerdir(filename)
207 if layerdir:
208 return self.get_layer_name(layerdir)
209 else:
210 return '?'
211
212 def get_file_layerdir(self, filename):
213 layer = bb.utils.get_file_layer(filename, self.tinfoil.config_data)
214 return self.bbfile_collections.get(layer, None)
215
216 def apply_append(self, appendname, recipename):
217 with open(appendname, 'r') as appendfile:
218 with open(recipename, 'a') as recipefile:
219 recipefile.write('\n')
220 recipefile.write('##### bbappended from %s #####\n' % self.get_file_layer(appendname))
221 recipefile.writelines(appendfile.readlines())
222
223 def register_commands(self, sp):
224 parser_add_layer = self.add_command(sp, 'add-layer', self.do_add_layer, parserecipes=False)
225 parser_add_layer.add_argument('layerdir', help='Layer directory to add')
226
227 parser_remove_layer = self.add_command(sp, 'remove-layer', self.do_remove_layer, parserecipes=False)
228 parser_remove_layer.add_argument('layerdir', help='Layer directory to remove (wildcards allowed, enclose in quotes to avoid shell expansion)')
229 parser_remove_layer.set_defaults(func=self.do_remove_layer)
230
231 parser_flatten = self.add_command(sp, 'flatten', self.do_flatten)
232 parser_flatten.add_argument('layer', nargs='*', help='Optional layer(s) to flatten (otherwise all are flattened)')
233 parser_flatten.add_argument('outputdir', help='Output directory')